From 677ff5d137f344f12d17f2b79e32dc9b4150697c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Mon, 18 Sep 2023 11:08:25 +0200 Subject: [PATCH 01/29] Move blob key and blob iv definition out of internal package A public interface of blend layer used internal key structure which made it impossible to be used outside of cinode repo. --- pkg/blenc/datastore.go | 7 +++---- pkg/blenc/datastore_dynamic_link.go | 9 ++++----- pkg/blenc/datastore_static.go | 10 +++++----- pkg/blenc/interface.go | 9 ++++----- pkg/blenc/interface_test.go | 4 ++-- .../cipherfactory/types.go => common/blob_keys.go} | 8 ++++---- pkg/internal/blobtypes/dynamiclink/public.go | 4 ++-- pkg/internal/blobtypes/dynamiclink/vectors_test.go | 3 +-- .../utilities/cipherfactory/cipher_factory.go | 9 +++++---- pkg/internal/utilities/cipherfactory/generator.go | 14 +++++++------- .../utilities/cipherfactory/generator_test.go | 6 +++--- pkg/structure/cinodefs.go | 3 +-- pkg/structure/link.go | 3 +-- 13 files changed, 42 insertions(+), 47 deletions(-) rename pkg/{internal/utilities/cipherfactory/types.go => common/blob_keys.go} (84%) diff --git a/pkg/blenc/datastore.go b/pkg/blenc/datastore.go index e10fd26..edc9a27 100644 --- a/pkg/blenc/datastore.go +++ b/pkg/blenc/datastore.go @@ -25,7 +25,6 @@ import ( "github.com/cinode/go/pkg/blobtypes" "github.com/cinode/go/pkg/common" "github.com/cinode/go/pkg/datastore" - "github.com/cinode/go/pkg/internal/utilities/cipherfactory" "github.com/cinode/go/pkg/internal/utilities/securefifo" ) @@ -51,7 +50,7 @@ type beDatastore struct { newSecureFifo secureFifoGenerator } -func (be *beDatastore) Open(ctx context.Context, name common.BlobName, key cipherfactory.Key) (io.ReadCloser, error) { +func (be *beDatastore) Open(ctx context.Context, name common.BlobName, key common.BlobKey) (io.ReadCloser, error) { switch name.Type() { case blobtypes.Static: return be.openStatic(ctx, name, key) @@ -67,7 +66,7 @@ func (be *beDatastore) Create( r io.Reader, ) ( common.BlobName, - cipherfactory.Key, + common.BlobKey, AuthInfo, error, ) { @@ -80,7 +79,7 @@ func (be *beDatastore) Create( return nil, nil, nil, blobtypes.ErrUnknownBlobType } -func (be *beDatastore) Update(ctx context.Context, name common.BlobName, authInfo AuthInfo, key cipherfactory.Key, r io.Reader) error { +func (be *beDatastore) Update(ctx context.Context, name common.BlobName, authInfo AuthInfo, key common.BlobKey, r io.Reader) error { switch name.Type() { case blobtypes.Static: return be.updateStatic(ctx, name, authInfo, key, r) diff --git a/pkg/blenc/datastore_dynamic_link.go b/pkg/blenc/datastore_dynamic_link.go index 8935911..fde273d 100644 --- a/pkg/blenc/datastore_dynamic_link.go +++ b/pkg/blenc/datastore_dynamic_link.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -25,7 +25,6 @@ import ( "github.com/cinode/go/pkg/common" "github.com/cinode/go/pkg/internal/blobtypes/dynamiclink" - "github.com/cinode/go/pkg/internal/utilities/cipherfactory" ) var ( @@ -38,7 +37,7 @@ var ( func (be *beDatastore) openDynamicLink( ctx context.Context, name common.BlobName, - key cipherfactory.Key, + key common.BlobKey, ) ( io.ReadCloser, error, @@ -78,7 +77,7 @@ func (be *beDatastore) createDynamicLink( r io.Reader, ) ( common.BlobName, - cipherfactory.Key, + common.BlobKey, AuthInfo, error, ) { @@ -111,7 +110,7 @@ func (be *beDatastore) updateDynamicLink( ctx context.Context, name common.BlobName, authInfo AuthInfo, - key cipherfactory.Key, + key common.BlobKey, r io.Reader, ) error { newVersion := be.generateVersion() diff --git a/pkg/blenc/datastore_static.go b/pkg/blenc/datastore_static.go index d28dec8..66bc808 100644 --- a/pkg/blenc/datastore_static.go +++ b/pkg/blenc/datastore_static.go @@ -33,14 +33,14 @@ var ( ErrCanNotUpdateStaticBlob = errors.New("blob update is not supported for static blobs") ) -func (be *beDatastore) openStatic(ctx context.Context, name common.BlobName, key cipherfactory.Key) (io.ReadCloser, error) { +func (be *beDatastore) openStatic(ctx context.Context, name common.BlobName, key common.BlobKey) (io.ReadCloser, error) { rc, err := be.ds.Open(ctx, name) if err != nil { return nil, err } - scr, err := cipherfactory.StreamCipherReader(key, key.DefaultIV(), rc) + scr, err := cipherfactory.StreamCipherReader(key, cipherfactory.DefaultIV(key), rc) if err != nil { return nil, err } @@ -69,7 +69,7 @@ func (be *beDatastore) createStatic( r io.Reader, ) ( common.BlobName, - cipherfactory.Key, + common.BlobKey, AuthInfo, error, ) { @@ -92,7 +92,7 @@ func (be *beDatastore) createStatic( } key := keyGenerator.Generate() - iv := key.DefaultIV() // We can use this since each blob will have different key + iv := cipherfactory.DefaultIV(key) // We can use this since each blob will have different key rClone, err := tempWriteBufferPlain.Done() // rClone will allow re-reading the source data if err != nil { @@ -143,7 +143,7 @@ func (be *beDatastore) updateStatic( ctx context.Context, name common.BlobName, authInfo AuthInfo, - key cipherfactory.Key, + key common.BlobKey, r io.Reader, ) error { return ErrCanNotUpdateStaticBlob diff --git a/pkg/blenc/interface.go b/pkg/blenc/interface.go index d26b29b..65ccb51 100644 --- a/pkg/blenc/interface.go +++ b/pkg/blenc/interface.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,7 +22,6 @@ import ( "github.com/cinode/go/pkg/common" "github.com/cinode/go/pkg/datastore" - "github.com/cinode/go/pkg/internal/utilities/cipherfactory" ) // AuthInfo is an opaque data that is necessary to perform update of a blob with the same name @@ -40,17 +39,17 @@ type BE interface { // // If returned error is not nil, the reader must be nil. Otherwise it is required to // close the reader once done working with it. - Open(ctx context.Context, name common.BlobName, key cipherfactory.Key) (io.ReadCloser, error) + Open(ctx context.Context, name common.BlobName, key common.BlobKey) (io.ReadCloser, error) // Create completely new blob with given dataset, as a result, the blob name and optional // AuthInfo that allows blob's update is returned - Create(ctx context.Context, blobType common.BlobType, r io.Reader) (common.BlobName, cipherfactory.Key, AuthInfo, error) + Create(ctx context.Context, blobType common.BlobType, r io.Reader) (common.BlobName, common.BlobKey, AuthInfo, error) // Update updates given blob type with new data, // The update must happen within a single blob name (i.e. it can not end up with blob with different name) // and may not be available for certain blob types such as static blobs. // A valid auth info is necessary to ensure a correct new content can be created - Update(ctx context.Context, name common.BlobName, ai AuthInfo, key cipherfactory.Key, r io.Reader) error + Update(ctx context.Context, name common.BlobName, ai AuthInfo, key common.BlobKey, r io.Reader) error // Exists does check whether blob of given name exists. It forwards the call // to underlying datastore. diff --git a/pkg/blenc/interface_test.go b/pkg/blenc/interface_test.go index 08f6334..fb10e16 100644 --- a/pkg/blenc/interface_test.go +++ b/pkg/blenc/interface_test.go @@ -261,13 +261,13 @@ func (s *BlencTestSuite) TestInvalidBlobTypes() { }) s.Run("must fail to open blob of invalid type", func() { - rc, err := s.be.Open(context.Background(), invalidBlobName, cipherfactory.Key{}) + rc, err := s.be.Open(context.Background(), invalidBlobName, common.BlobKey{}) s.Require().ErrorIs(err, blobtypes.ErrUnknownBlobType) s.Require().Nil(rc) }) s.Run("must fail to update blob of invalid type", func() { - err = s.be.Update(context.Background(), invalidBlobName, AuthInfo{}, cipherfactory.Key{}, bytes.NewReader(nil)) + err = s.be.Update(context.Background(), invalidBlobName, AuthInfo{}, common.BlobKey{}, bytes.NewReader(nil)) s.Require().ErrorIs(err, blobtypes.ErrUnknownBlobType) }) } diff --git a/pkg/internal/utilities/cipherfactory/types.go b/pkg/common/blob_keys.go similarity index 84% rename from pkg/internal/utilities/cipherfactory/types.go rename to pkg/common/blob_keys.go index af03f8b..f58ec10 100644 --- a/pkg/internal/utilities/cipherfactory/types.go +++ b/pkg/common/blob_keys.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Bartłomiej Święcki (byo) +Copyright © 2022 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,10 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cipherfactory +package common // Key with cipher type -type Key []byte +type BlobKey []byte // IV -type IV []byte +type BlobIV []byte diff --git a/pkg/internal/blobtypes/dynamiclink/public.go b/pkg/internal/blobtypes/dynamiclink/public.go index 5d8cbbe..1568faf 100644 --- a/pkg/internal/blobtypes/dynamiclink/public.go +++ b/pkg/internal/blobtypes/dynamiclink/public.go @@ -244,7 +244,7 @@ func (d *PublicReader) ivGeneratorPrefilled() cipherfactory.IVGenerator { return ivGenerator } -func (d *PublicReader) validateKeyInLinkData(key cipherfactory.Key, r io.Reader) error { +func (d *PublicReader) validateKeyInLinkData(key common.BlobKey, r io.Reader) error { // At the beginning of the data there's the key validation block, // that block contains a proof that the encryption key was deterministically derived // from the blob name (thus preventing weak key attack) @@ -276,7 +276,7 @@ func (d *PublicReader) validateKeyInLinkData(key cipherfactory.Key, r io.Reader) return nil } -func (d *PublicReader) GetLinkDataReader(key cipherfactory.Key) (io.Reader, error) { +func (d *PublicReader) GetLinkDataReader(key common.BlobKey) (io.Reader, error) { r, err := cipherfactory.StreamCipherReader(key, d.iv, d.GetEncryptedLinkReader()) if err != nil { diff --git a/pkg/internal/blobtypes/dynamiclink/vectors_test.go b/pkg/internal/blobtypes/dynamiclink/vectors_test.go index 8280ec5..7bcbe9f 100644 --- a/pkg/internal/blobtypes/dynamiclink/vectors_test.go +++ b/pkg/internal/blobtypes/dynamiclink/vectors_test.go @@ -27,7 +27,6 @@ import ( "testing" "github.com/cinode/go/pkg/common" - "github.com/cinode/go/pkg/internal/utilities/cipherfactory" "github.com/stretchr/testify/require" ) @@ -104,7 +103,7 @@ func TestVectors(t *testing.T) { return err } - dr, err := pr.GetLinkDataReader(cipherfactory.Key(testCase.EncryptionKey)) + dr, err := pr.GetLinkDataReader(common.BlobKey(testCase.EncryptionKey)) if err != nil { return err } diff --git a/pkg/internal/utilities/cipherfactory/cipher_factory.go b/pkg/internal/utilities/cipherfactory/cipher_factory.go index 7d17dcb..9ee69c8 100644 --- a/pkg/internal/utilities/cipherfactory/cipher_factory.go +++ b/pkg/internal/utilities/cipherfactory/cipher_factory.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ import ( "fmt" "io" + "github.com/cinode/go/pkg/common" "golang.org/x/crypto/chacha20" ) @@ -39,7 +40,7 @@ const ( reservedByteForKeyType byte = 0 ) -func StreamCipherReader(key Key, iv IV, r io.Reader) (io.Reader, error) { +func StreamCipherReader(key common.BlobKey, iv common.BlobIV, r io.Reader) (io.Reader, error) { stream, err := _cipherForKeyIV(key, iv) if err != nil { return nil, err @@ -47,7 +48,7 @@ func StreamCipherReader(key Key, iv IV, r io.Reader) (io.Reader, error) { return &cipher.StreamReader{S: stream, R: r}, nil } -func StreamCipherWriter(key Key, iv IV, w io.Writer) (io.Writer, error) { +func StreamCipherWriter(key common.BlobKey, iv common.BlobIV, w io.Writer) (io.Writer, error) { stream, err := _cipherForKeyIV(key, iv) if err != nil { return nil, err @@ -55,7 +56,7 @@ func StreamCipherWriter(key Key, iv IV, w io.Writer) (io.Writer, error) { return cipher.StreamWriter{S: stream, W: w}, nil } -func _cipherForKeyIV(key Key, iv IV) (cipher.Stream, error) { +func _cipherForKeyIV(key common.BlobKey, iv common.BlobIV) (cipher.Stream, error) { if len(key) == 0 || key[0] != reservedByteForKeyType { return nil, ErrInvalidEncryptionConfigKeyType } diff --git a/pkg/internal/utilities/cipherfactory/generator.go b/pkg/internal/utilities/cipherfactory/generator.go index 50caa71..6c07cb4 100644 --- a/pkg/internal/utilities/cipherfactory/generator.go +++ b/pkg/internal/utilities/cipherfactory/generator.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -33,7 +33,7 @@ const ( type KeyGenerator interface { io.Writer - Generate() Key + Generate() common.BlobKey } type keyGenerator struct { @@ -42,7 +42,7 @@ type keyGenerator struct { func (g keyGenerator) Write(b []byte) (int, error) { return g.h.Write(b) } -func (g keyGenerator) Generate() Key { +func (g keyGenerator) Generate() common.BlobKey { return append( []byte{reservedByteForKeyType}, g.h.Sum(nil)[:chacha20.KeySize]..., @@ -51,7 +51,7 @@ func (g keyGenerator) Generate() Key { type IVGenerator interface { io.Writer - Generate() IV + Generate() common.BlobIV } type ivGenerator struct { @@ -60,7 +60,7 @@ type ivGenerator struct { func (g ivGenerator) Write(b []byte) (int, error) { return g.h.Write(b) } -func (g ivGenerator) Generate() IV { +func (g ivGenerator) Generate() common.BlobIV { return g.h.Sum(nil)[:chacha20.NonceSizeX] } @@ -77,12 +77,12 @@ func NewIVGenerator(t common.BlobType) IVGenerator { return ivGenerator{h: h} } -var defaultXChaCha20IV = func() IV { +var defaultXChaCha20IV = func() common.BlobIV { h := sha256.New() h.Write([]byte{preambleHashDefaultIV, reservedByteForKeyType}) return h.Sum(nil)[:chacha20.NonceSizeX] }() -func (k Key) DefaultIV() IV { +func DefaultIV(k common.BlobKey) common.BlobIV { return defaultXChaCha20IV } diff --git a/pkg/internal/utilities/cipherfactory/generator_test.go b/pkg/internal/utilities/cipherfactory/generator_test.go index cd146d6..316d1ef 100644 --- a/pkg/internal/utilities/cipherfactory/generator_test.go +++ b/pkg/internal/utilities/cipherfactory/generator_test.go @@ -46,7 +46,7 @@ func TestGenerator(t *testing.T) { _, err = _cipherForKeyIV(key, iv) require.NoError(t, err) - _, err = _cipherForKeyIV(key, key.DefaultIV()) + _, err = _cipherForKeyIV(key, DefaultIV(key)) require.NoError(t, err) // Check initial bytes of keys only - since key and IV are of different @@ -56,8 +56,8 @@ func TestGenerator(t *testing.T) { // be using same hashed dataset which may be exploitable since IV // is made public require.NotEqual(t, key[1:1+8], iv[:8]) - require.NotEqual(t, key[1:1+8], key.DefaultIV()[:8]) - require.NotEqual(t, iv[:8], key.DefaultIV()[:8]) + require.NotEqual(t, key[1:1+8], DefaultIV(key)[:8]) + require.NotEqual(t, iv[:8], DefaultIV(key)[:8]) // Note: once other key types are introduced, we should also check // that for different key types there are different hashes diff --git a/pkg/structure/cinodefs.go b/pkg/structure/cinodefs.go index e9e9cd4..4c863a7 100644 --- a/pkg/structure/cinodefs.go +++ b/pkg/structure/cinodefs.go @@ -24,7 +24,6 @@ import ( "github.com/cinode/go/pkg/blenc" "github.com/cinode/go/pkg/common" - "github.com/cinode/go/pkg/internal/utilities/cipherfactory" "github.com/cinode/go/pkg/protobuf" "google.golang.org/protobuf/proto" ) @@ -40,7 +39,7 @@ func (d *CinodeFS) OpenContent(ctx context.Context, ep *protobuf.Entrypoint) (io return d.BE.Open( ctx, common.BlobName(ep.BlobName), - cipherfactory.Key(ep.GetKeyInfo().GetKey()), + common.BlobKey(ep.GetKeyInfo().GetKey()), ) } diff --git a/pkg/structure/link.go b/pkg/structure/link.go index e9d2b44..36070c6 100644 --- a/pkg/structure/link.go +++ b/pkg/structure/link.go @@ -26,7 +26,6 @@ import ( "github.com/cinode/go/pkg/blenc" "github.com/cinode/go/pkg/blobtypes" "github.com/cinode/go/pkg/common" - "github.com/cinode/go/pkg/internal/utilities/cipherfactory" "github.com/cinode/go/pkg/protobuf" ) @@ -104,7 +103,7 @@ func DereferenceLink( rc, err := be.Open( ctx, common.BlobName(link.BlobName), - cipherfactory.Key(link.GetKeyInfo().GetKey()), + common.BlobKey(link.GetKeyInfo().GetKey()), ) if err != nil { return nil, err From e90b36eec5e9c460ad257e93593914714ec50571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Mon, 18 Sep 2023 11:08:40 +0200 Subject: [PATCH 02/29] Minor vscode dict udpate --- .vscode/settings.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index 18ce17c..3f5bd96 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,6 +6,7 @@ "blobtype", "blobtypes", "bmap", + "chacha", "cinode", "cinodefs", "cipherfactory", @@ -15,6 +16,7 @@ "dynamiclink", "elink", "fifos", + "fsys", "goveralls", "Hasher", "jbenet", From 5b57ba2686faa2637d7d39a016949c00b2e501f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Mon, 18 Sep 2023 21:44:14 +0200 Subject: [PATCH 03/29] Convert common.BlobKey and common.BlobIV to dedicated structs This change hides the internal data of BlobKey and BlobIV types to ensure those internals can not be relied on in outside code. --- pkg/blenc/datastore.go | 2 +- pkg/blenc/datastore_dynamic_link.go | 8 ++-- pkg/blenc/datastore_dynamic_link_test.go | 4 +- pkg/blenc/datastore_static.go | 21 +++++----- pkg/blenc/datastore_static_test.go | 12 +++--- pkg/blenc/interface_test.go | 25 ++++++++---- pkg/cmd/cinode_web_proxy/root_test.go | 2 +- pkg/common/blob_keys.go | 25 ++++++++++-- pkg/common/blob_keys_test.go | 39 +++++++++++++++++++ pkg/internal/blobtypes/dynamiclink/public.go | 11 +++--- .../blobtypes/dynamiclink/public_test.go | 6 ++- .../blobtypes/dynamiclink/publisher.go | 13 ++++--- .../blobtypes/dynamiclink/publisher_test.go | 5 ++- .../blobtypes/dynamiclink/vectors_test.go | 4 +- .../utilities/cipherfactory/cipher_factory.go | 14 ++++--- .../cipherfactory/cipher_factory_test.go | 19 ++++++--- .../utilities/cipherfactory/generator.go | 8 ++-- .../utilities/cipherfactory/generator_test.go | 9 +++-- pkg/structure/cinodefs.go | 2 +- pkg/structure/directory.go | 8 ++-- pkg/structure/link.go | 14 +++++-- 21 files changed, 173 insertions(+), 78 deletions(-) create mode 100644 pkg/common/blob_keys_test.go diff --git a/pkg/blenc/datastore.go b/pkg/blenc/datastore.go index edc9a27..1f47caa 100644 --- a/pkg/blenc/datastore.go +++ b/pkg/blenc/datastore.go @@ -76,7 +76,7 @@ func (be *beDatastore) Create( case blobtypes.DynamicLink: return be.createDynamicLink(ctx, r) } - return nil, nil, nil, blobtypes.ErrUnknownBlobType + return nil, common.BlobKey{}, nil, blobtypes.ErrUnknownBlobType } func (be *beDatastore) Update(ctx context.Context, name common.BlobName, authInfo AuthInfo, key common.BlobKey, r io.Reader) error { diff --git a/pkg/blenc/datastore_dynamic_link.go b/pkg/blenc/datastore_dynamic_link.go index fde273d..6c42dc1 100644 --- a/pkg/blenc/datastore_dynamic_link.go +++ b/pkg/blenc/datastore_dynamic_link.go @@ -85,19 +85,19 @@ func (be *beDatastore) createDynamicLink( dl, err := dynamiclink.Create(be.rand) if err != nil { - return nil, nil, nil, err + return nil, common.BlobKey{}, nil, err } pr, encryptionKey, err := dl.UpdateLinkData(r, version) if err != nil { - return nil, nil, nil, err + return nil, common.BlobKey{}, nil, err } // Send update packet bn := dl.BlobName() err = be.ds.Update(ctx, bn, pr.GetPublicDataReader()) if err != nil { - return nil, nil, nil, err + return nil, common.BlobKey{}, nil, err } return bn, @@ -126,7 +126,7 @@ func (be *beDatastore) updateDynamicLink( } // Sanity checks - if !bytes.Equal(encryptionKey, key) { + if !encryptionKey.Equal(key) { return ErrDynamicLinkUpdateFailedWrongKey } if !bytes.Equal(name, dl.BlobName()) { diff --git a/pkg/blenc/datastore_dynamic_link_test.go b/pkg/blenc/datastore_dynamic_link_test.go index f9777d4..8e1b12f 100644 --- a/pkg/blenc/datastore_dynamic_link_test.go +++ b/pkg/blenc/datastore_dynamic_link_test.go @@ -125,7 +125,7 @@ func TestDynamicLinkErrors(t *testing.T) { bn, key, ai, err := be.Create(context.Background(), blobtypes.DynamicLink, bytes.NewReader(nil)) require.ErrorIs(t, err, injectedErr) require.Nil(t, bn) - require.Nil(t, key) + require.Equal(t, common.BlobKey{}, key) require.Nil(t, ai) be.(*beDatastore).rand = rand.Reader @@ -140,7 +140,7 @@ func TestDynamicLinkErrors(t *testing.T) { bn, key, ai, err := be.Create(context.Background(), blobtypes.DynamicLink, bytes.NewReader(nil)) require.ErrorIs(t, err, injectedErr) require.Nil(t, bn) - require.Nil(t, key) + require.Equal(t, common.BlobKey{}, key) require.Nil(t, ai) dsw.updateFn = nil diff --git a/pkg/blenc/datastore_static.go b/pkg/blenc/datastore_static.go index 66bc808..4ebac95 100644 --- a/pkg/blenc/datastore_static.go +++ b/pkg/blenc/datastore_static.go @@ -17,7 +17,6 @@ limitations under the License. package blenc import ( - "bytes" "context" "crypto/sha256" "errors" @@ -54,7 +53,7 @@ func (be *beDatastore) openStatic(ctx context.Context, name common.BlobName, key Reader: validatingreader.CheckOnEOF( io.TeeReader(scr, keyGenerator), func() error { - if !bytes.Equal(key, keyGenerator.Generate()) { + if !key.Equal(keyGenerator.Generate()) { return blobtypes.ErrValidationFailed } return nil @@ -75,20 +74,20 @@ func (be *beDatastore) createStatic( ) { tempWriteBufferPlain, err := be.newSecureFifo() if err != nil { - return nil, nil, nil, err + return nil, common.BlobKey{}, nil, err } defer tempWriteBufferPlain.Close() tempWriteBufferEncrypted, err := be.newSecureFifo() if err != nil { - return nil, nil, nil, err + return nil, common.BlobKey{}, nil, err } defer tempWriteBufferEncrypted.Close() keyGenerator := cipherfactory.NewKeyGenerator(blobtypes.Static) _, err = io.Copy(tempWriteBufferPlain, io.TeeReader(r, keyGenerator)) if err != nil { - return nil, nil, nil, err + return nil, common.BlobKey{}, nil, err } key := keyGenerator.Generate() @@ -96,7 +95,7 @@ func (be *beDatastore) createStatic( rClone, err := tempWriteBufferPlain.Done() // rClone will allow re-reading the source data if err != nil { - return nil, nil, nil, err + return nil, common.BlobKey{}, nil, err } defer rClone.Close() @@ -110,30 +109,30 @@ func (be *beDatastore) createStatic( ), ) if err != nil { - return nil, nil, nil, err + return nil, common.BlobKey{}, nil, err } _, err = io.Copy(encWriter, rClone) if err != nil { - return nil, nil, nil, err + return nil, common.BlobKey{}, nil, err } encReader, err := tempWriteBufferEncrypted.Done() if err != nil { - return nil, nil, nil, err + return nil, common.BlobKey{}, nil, err } defer encReader.Close() // Generate blob name from the encrypted data name, err := common.BlobNameFromHashAndType(blobNameHasher.Sum(nil), blobtypes.Static) if err != nil { - return nil, nil, nil, err + return nil, common.BlobKey{}, nil, err } // Send encrypted blob into the datastore err = be.ds.Update(ctx, name, encReader) if err != nil { - return nil, nil, nil, err + return nil, common.BlobKey{}, nil, err } return name, key, nil, nil diff --git a/pkg/blenc/datastore_static_test.go b/pkg/blenc/datastore_static_test.go index bf370be..82747ef 100644 --- a/pkg/blenc/datastore_static_test.go +++ b/pkg/blenc/datastore_static_test.go @@ -98,7 +98,7 @@ func TestStaticErrorTruncatedDatastore(t *testing.T) { bn, key, ai, err := be.Create(context.Background(), blobtypes.Static, bytes.NewReader(nil)) require.ErrorIs(t, err, injectedErr) require.Nil(t, bn) - require.Nil(t, key) + require.Equal(t, common.BlobKey{}, key) require.Nil(t, ai) }) @@ -128,7 +128,7 @@ func TestStaticErrorTruncatedDatastore(t *testing.T) { bn, key, ai, err := be.Create(context.Background(), blobtypes.Static, bytes.NewReader(nil)) require.ErrorIs(t, err, injectedErr) require.Nil(t, bn) - require.Nil(t, key) + require.Equal(t, common.BlobKey{}, key) require.Nil(t, ai) require.True(t, firstSecureFifoCreated) require.True(t, firstSecureFifoClosed) @@ -166,7 +166,7 @@ func TestStaticErrorTruncatedDatastore(t *testing.T) { bn, key, ai, err := be.Create(context.Background(), blobtypes.Static, bytes.NewReader(nil)) require.ErrorIs(t, err, injectedErr) require.Nil(t, bn) - require.Nil(t, key) + require.Equal(t, common.BlobKey{}, key) require.Nil(t, ai) require.Equal(t, 2, secureFifosCreated) require.Equal(t, secureFifosCreated, secureFifosClosed) @@ -205,7 +205,7 @@ func TestStaticErrorTruncatedDatastore(t *testing.T) { bn, key, ai, err := be.Create(context.Background(), blobtypes.Static, bytes.NewReader([]byte("Hello world"))) require.ErrorIs(t, err, injectedErr) require.Nil(t, bn) - require.Nil(t, key) + require.Equal(t, common.BlobKey{}, key) require.Nil(t, ai) require.Equal(t, 2, secureFifosCreated) require.Equal(t, secureFifosCreated, secureFifosClosed) @@ -237,7 +237,7 @@ func TestStaticErrorTruncatedDatastore(t *testing.T) { bn, key, ai, err := be.Create(context.Background(), blobtypes.Static, iotest.ErrReader(injectedErr)) require.ErrorIs(t, err, injectedErr) require.Nil(t, bn) - require.Nil(t, key) + require.Equal(t, common.BlobKey{}, key) require.Nil(t, ai) require.Equal(t, 2, secureFifosCreated) require.Equal(t, secureFifosCreated, secureFifosClosed) @@ -272,7 +272,7 @@ func TestStaticErrorTruncatedDatastore(t *testing.T) { bn, key, ai, err := be.Create(context.Background(), blobtypes.Static, bytes.NewReader(nil)) require.ErrorIs(t, err, injectedErr) require.Nil(t, bn) - require.Nil(t, key) + require.Equal(t, common.BlobKey{}, key) require.Nil(t, ai) require.Equal(t, 2, secureFifosCreated) diff --git a/pkg/blenc/interface_test.go b/pkg/blenc/interface_test.go index fb10e16..39eaf41 100644 --- a/pkg/blenc/interface_test.go +++ b/pkg/blenc/interface_test.go @@ -93,7 +93,7 @@ func (s *BlencTestSuite) TestStaticBlobs() { s.Run("new static blob must be different from the first one", func() { s.Require().NoError(err) s.Require().NotEqual(key, key2) - s.Require().Len(key2, len(key)) + s.Require().Len(key2.Bytes(), len(key.Bytes())) }) s.Run("must fail to update static blob", func() { @@ -121,7 +121,8 @@ func (s *BlencTestSuite) TestStaticBlobs() { }) s.Run("must fail to open static blob with invalid key", func() { - rc, err := s.be.Open(context.Background(), bn2, key2[1:]) + brokenKey := common.BlobKeyFromBytes(key2.Bytes()[1:]) + rc, err := s.be.Open(context.Background(), bn2, brokenKey) s.Require().ErrorIs(err, cipherfactory.ErrInvalidEncryptionConfig) s.Require().Nil(rc) }) @@ -179,7 +180,7 @@ func (s *BlencTestSuite) TestDynamicLinkSuccessPath() { s.Run("new dynamic link must be different from the first one", func() { s.Require().NoError(err) s.Require().NotEqual(key, key2) - s.Require().Len(key2, len(key)) + s.Require().Len(key2.Bytes(), len(key.Bytes())) }) s.Run("must correctly read blob's content", func() { @@ -243,7 +244,7 @@ func (s *BlencTestSuite) TestDynamicLinkSuccessPath() { bn, key, ai, err := s.be.Create(context.Background(), blobtypes.DynamicLink, iotest.ErrReader(injectedErr)) s.Require().ErrorIs(err, injectedErr) s.Require().Nil(bn) - s.Require().Nil(key) + s.Require().Equal(common.BlobKey{}, key) s.Require().Nil(ai) }) } @@ -256,18 +257,28 @@ func (s *BlencTestSuite) TestInvalidBlobTypes() { bn, key, ai, err := s.be.Create(context.Background(), blobtypes.Invalid, bytes.NewReader(nil)) s.Require().ErrorIs(err, blobtypes.ErrUnknownBlobType) s.Require().Nil(bn) - s.Require().Nil(key) + s.Require().Equal(common.BlobKey{}, key) s.Require().Nil(ai) }) s.Run("must fail to open blob of invalid type", func() { - rc, err := s.be.Open(context.Background(), invalidBlobName, common.BlobKey{}) + rc, err := s.be.Open( + context.Background(), + invalidBlobName, + common.BlobKey{}, + ) s.Require().ErrorIs(err, blobtypes.ErrUnknownBlobType) s.Require().Nil(rc) }) s.Run("must fail to update blob of invalid type", func() { - err = s.be.Update(context.Background(), invalidBlobName, AuthInfo{}, common.BlobKey{}, bytes.NewReader(nil)) + err = s.be.Update( + context.Background(), + invalidBlobName, + AuthInfo{}, + common.BlobKey{}, + bytes.NewReader(nil), + ) s.Require().ErrorIs(err, blobtypes.ErrUnknownBlobType) }) } diff --git a/pkg/cmd/cinode_web_proxy/root_test.go b/pkg/cmd/cinode_web_proxy/root_test.go index 68c336a..98aef04 100644 --- a/pkg/cmd/cinode_web_proxy/root_test.go +++ b/pkg/cmd/cinode_web_proxy/root_test.go @@ -118,7 +118,7 @@ func TestWebProxyHandlerInvalidEntrypoint(t *testing.T) { BlobName: n, MimeType: structure.CinodeDirMimeType, KeyInfo: &protobuf.KeyInfo{ - Key: cipherfactory.NewKeyGenerator(blobtypes.Static).Generate(), + Key: cipherfactory.NewKeyGenerator(blobtypes.Static).Generate().Bytes(), }, }, ) diff --git a/pkg/common/blob_keys.go b/pkg/common/blob_keys.go index f58ec10..5518874 100644 --- a/pkg/common/blob_keys.go +++ b/pkg/common/blob_keys.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,8 +16,27 @@ limitations under the License. package common +import "crypto/subtle" + +func copyBytes(b []byte) []byte { + if b == nil { + return nil + } + ret := make([]byte, len(b)) + copy(ret, b) + return ret +} + // Key with cipher type -type BlobKey []byte +type BlobKey struct{ key []byte } + +func BlobKeyFromBytes(key []byte) BlobKey { return BlobKey{key: copyBytes(key)} } +func (k BlobKey) Bytes() []byte { return copyBytes(k.key) } +func (k BlobKey) Equal(k2 BlobKey) bool { return subtle.ConstantTimeCompare(k.key, k2.key) == 1 } // IV -type BlobIV []byte +type BlobIV struct{ iv []byte } + +func BlobIVFromBytes(iv []byte) BlobIV { return BlobIV{iv: copyBytes(iv)} } +func (i BlobIV) Bytes() []byte { return copyBytes(i.iv) } +func (i BlobIV) Equal(i2 BlobIV) bool { return subtle.ConstantTimeCompare(i.iv, i2.iv) == 1 } diff --git a/pkg/common/blob_keys_test.go b/pkg/common/blob_keys_test.go new file mode 100644 index 0000000..171f7ae --- /dev/null +++ b/pkg/common/blob_keys_test.go @@ -0,0 +1,39 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBlobKey(t *testing.T) { + keyBytes := []byte{1, 2, 3} + key := BlobKeyFromBytes(keyBytes) + require.Equal(t, keyBytes, key.Bytes()) + require.True(t, key.Equal(BlobKeyFromBytes(keyBytes))) + require.Nil(t, BlobKey{}.Bytes()) +} + +func TestBlobIV(t *testing.T) { + ivBytes := []byte{1, 2, 3} + iv := BlobIVFromBytes(ivBytes) + require.Equal(t, ivBytes, iv.Bytes()) + require.True(t, iv.Equal(BlobIVFromBytes(ivBytes))) + require.Nil(t, BlobKey{}.Bytes()) +} diff --git a/pkg/internal/blobtypes/dynamiclink/public.go b/pkg/internal/blobtypes/dynamiclink/public.go index 1568faf..0002799 100644 --- a/pkg/internal/blobtypes/dynamiclink/public.go +++ b/pkg/internal/blobtypes/dynamiclink/public.go @@ -80,7 +80,7 @@ type PublicReader struct { Public contentVersion uint64 signature []byte - iv []byte + iv common.BlobIV r io.Reader } @@ -140,10 +140,11 @@ func FromPublicData(name common.BlobName, r io.Reader) (*PublicReader, error) { return nil, err } - dl.iv, err = readDynamicSizeBuff(r, "iv") + iv, err := readDynamicSizeBuff(r, "iv") if err != nil { return nil, err } + dl.iv = common.BlobIVFromBytes(iv) // Starting from validations at this point, errors are returned while reading. // This is to prepare for future improvements when real streaming is @@ -201,7 +202,7 @@ func (d *PublicReader) GetPublicDataReader() io.Reader { // Preamble - dynamic link data storeBuff(w, d.signature) storeUint64(w, d.contentVersion) - storeDynamicSizeBuff(w, d.iv) + storeDynamicSizeBuff(w, d.iv.Bytes()) return io.MultiReader( bytes.NewReader(w.Bytes()), // Preamble @@ -269,7 +270,7 @@ func (d *PublicReader) validateKeyInLinkData(key common.BlobKey, r io.Reader) er keyGenerator.Write(signature) generatedKey := keyGenerator.Generate() - if !bytes.Equal(generatedKey, key) { + if !generatedKey.Equal(key) { return ErrInvalidDynamicLinkKeyMismatch } @@ -305,7 +306,7 @@ func (d *PublicReader) GetLinkDataReader(key common.BlobKey) (io.Reader, error) return validatingreader.CheckOnEOF( r, func() error { - if !bytes.Equal(ivHasher.Generate(), d.iv) { + if !d.iv.Equal(ivHasher.Generate()) { return ErrInvalidDynamicLinkIVMismatch } diff --git a/pkg/internal/blobtypes/dynamiclink/public_test.go b/pkg/internal/blobtypes/dynamiclink/public_test.go index ae31481..b02d36d 100644 --- a/pkg/internal/blobtypes/dynamiclink/public_test.go +++ b/pkg/internal/blobtypes/dynamiclink/public_test.go @@ -307,7 +307,9 @@ func TestPublicReaderGetLinkDataReader(t *testing.T) { require.NoError(t, err) // Flip a single bit in IV - pr.iv[len(pr.iv)/2] ^= 0x80 + ivBytes := pr.iv.Bytes() + ivBytes[len(ivBytes)/2] ^= 0x80 + pr.iv = common.BlobIVFromBytes(ivBytes) // Because the IV is incorrect, key validation block that is encrypted will be invalid // thus the method will complain about key, not the IV that will fail first @@ -322,7 +324,7 @@ func TestPublicReaderGetLinkDataReader(t *testing.T) { pr, _, err := link.UpdateLinkData(bytes.NewReader([]byte("Hello world")), 0) require.NoError(t, err) - _, err = pr.GetLinkDataReader(nil) + _, err = pr.GetLinkDataReader(common.BlobKey{}) require.ErrorIs(t, err, cipherfactory.ErrInvalidEncryptionConfigKeyType) }) } diff --git a/pkg/internal/blobtypes/dynamiclink/publisher.go b/pkg/internal/blobtypes/dynamiclink/publisher.go index 8359b6f..63f5c0e 100644 --- a/pkg/internal/blobtypes/dynamiclink/publisher.go +++ b/pkg/internal/blobtypes/dynamiclink/publisher.go @@ -24,6 +24,7 @@ import ( "io" "github.com/cinode/go/pkg/blobtypes" + "github.com/cinode/go/pkg/common" "github.com/cinode/go/pkg/internal/utilities/cipherfactory" ) @@ -106,7 +107,7 @@ func (dl *Publisher) AuthInfo() []byte { return ret[:] } -func (dl *Publisher) calculateEncryptionKey() ([]byte, []byte) { +func (dl *Publisher) calculateEncryptionKey() (common.BlobKey, []byte) { dataSeed := append( []byte{signatureForEncryptionKeyGeneration}, dl.BlobName()..., @@ -122,7 +123,7 @@ func (dl *Publisher) calculateEncryptionKey() ([]byte, []byte) { return key, signature } -func (dl *Publisher) UpdateLinkData(r io.Reader, version uint64) (*PublicReader, []byte, error) { +func (dl *Publisher) UpdateLinkData(r io.Reader, version uint64) (*PublicReader, common.BlobKey, error) { encryptionKey, kvb := dl.calculateEncryptionKey() // key validation block precedes the link data @@ -132,7 +133,7 @@ func (dl *Publisher) UpdateLinkData(r io.Reader, version uint64) (*PublicReader, _, err := io.Copy(unencryptedLinkBuff, r) if err != nil { - return nil, nil, err + return nil, common.BlobKey{}, err } unencryptedLink := unencryptedLinkBuff.Bytes() @@ -150,17 +151,17 @@ func (dl *Publisher) UpdateLinkData(r io.Reader, version uint64) (*PublicReader, encryptedLinkBuff := bytes.NewBuffer(nil) w, err := cipherfactory.StreamCipherWriter(encryptionKey, pr.iv, encryptedLinkBuff) if err != nil { - return nil, nil, err + return nil, common.BlobKey{}, err } _, err = w.Write(unencryptedLink) if err != nil { - return nil, nil, err + return nil, common.BlobKey{}, err } signatureHasher := pr.toSignDataHasherPrefilled() storeUint64(signatureHasher, pr.contentVersion) - storeDynamicSizeBuff(signatureHasher, pr.iv) + storeDynamicSizeBuff(signatureHasher, pr.iv.Bytes()) signatureHasher.Write(encryptedLinkBuff.Bytes()) pr.signature = ed25519.Sign(dl.privKey, signatureHasher.Sum(nil)) diff --git a/pkg/internal/blobtypes/dynamiclink/publisher_test.go b/pkg/internal/blobtypes/dynamiclink/publisher_test.go index 7f15ca9..47e9687 100644 --- a/pkg/internal/blobtypes/dynamiclink/publisher_test.go +++ b/pkg/internal/blobtypes/dynamiclink/publisher_test.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ import ( "testing" "testing/iotest" + "github.com/cinode/go/pkg/common" "github.com/stretchr/testify/require" ) @@ -131,6 +132,6 @@ func TestPublisherUpdateLinkData(t *testing.T) { pr2, key2, err := dl.UpdateLinkData(iotest.ErrReader(injectedErr), 3) require.ErrorIs(t, err, injectedErr) require.Nil(t, pr2) - require.Nil(t, key2) + require.Equal(t, common.BlobKey{}, key2) }) } diff --git a/pkg/internal/blobtypes/dynamiclink/vectors_test.go b/pkg/internal/blobtypes/dynamiclink/vectors_test.go index 7bcbe9f..4303546 100644 --- a/pkg/internal/blobtypes/dynamiclink/vectors_test.go +++ b/pkg/internal/blobtypes/dynamiclink/vectors_test.go @@ -103,7 +103,9 @@ func TestVectors(t *testing.T) { return err } - dr, err := pr.GetLinkDataReader(common.BlobKey(testCase.EncryptionKey)) + dr, err := pr.GetLinkDataReader( + common.BlobKeyFromBytes(testCase.EncryptionKey), + ) if err != nil { return err } diff --git a/pkg/internal/utilities/cipherfactory/cipher_factory.go b/pkg/internal/utilities/cipherfactory/cipher_factory.go index 9ee69c8..171c2de 100644 --- a/pkg/internal/utilities/cipherfactory/cipher_factory.go +++ b/pkg/internal/utilities/cipherfactory/cipher_factory.go @@ -57,17 +57,19 @@ func StreamCipherWriter(key common.BlobKey, iv common.BlobIV, w io.Writer) (io.W } func _cipherForKeyIV(key common.BlobKey, iv common.BlobIV) (cipher.Stream, error) { - if len(key) == 0 || key[0] != reservedByteForKeyType { + keyBytes := key.Bytes() + if len(keyBytes) == 0 || keyBytes[0] != reservedByteForKeyType { return nil, ErrInvalidEncryptionConfigKeyType } - if len(key) != chacha20.KeySize+1 { - return nil, fmt.Errorf("%w, got %d bytes", ErrInvalidEncryptionConfigKeySize, len(key)+1) + if len(keyBytes) != chacha20.KeySize+1 { + return nil, fmt.Errorf("%w, got %d bytes", ErrInvalidEncryptionConfigKeySize, len(keyBytes)+1) } - if len(iv) != chacha20.NonceSizeX { - return nil, fmt.Errorf("%w, got %d bytes", ErrInvalidEncryptionConfigIVSize, len(iv)) + ivBytes := iv.Bytes() + if len(ivBytes) != chacha20.NonceSizeX { + return nil, fmt.Errorf("%w, got %d bytes", ErrInvalidEncryptionConfigIVSize, len(ivBytes)) } - return chacha20.NewUnauthenticatedCipher(key[1:], iv) + return chacha20.NewUnauthenticatedCipher(keyBytes[1:], ivBytes) } diff --git a/pkg/internal/utilities/cipherfactory/cipher_factory_test.go b/pkg/internal/utilities/cipherfactory/cipher_factory_test.go index ea29b26..2a19954 100644 --- a/pkg/internal/utilities/cipherfactory/cipher_factory_test.go +++ b/pkg/internal/utilities/cipherfactory/cipher_factory_test.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import ( "io" "testing" + "github.com/cinode/go/pkg/common" "github.com/stretchr/testify/require" "golang.org/x/crypto/chacha20" ) @@ -59,13 +60,21 @@ func TestCipherForKeyIV(t *testing.T) { }, } { t.Run(d.desc, func(t *testing.T) { - sr, err := StreamCipherReader(d.key, d.iv, bytes.NewReader([]byte{})) + sr, err := StreamCipherReader( + common.BlobKeyFromBytes(d.key), + common.BlobIVFromBytes(d.iv), + bytes.NewReader([]byte{}), + ) require.ErrorIs(t, err, d.err) if err == nil { require.NotNil(t, sr) } - sw, err := StreamCipherWriter(d.key, d.iv, bytes.NewBuffer(nil)) + sw, err := StreamCipherWriter( + common.BlobKeyFromBytes(d.key), + common.BlobIVFromBytes(d.iv), + bytes.NewBuffer(nil), + ) require.ErrorIs(t, err, d.err) if err == nil { require.NotNil(t, sw) @@ -75,8 +84,8 @@ func TestCipherForKeyIV(t *testing.T) { } func TestStreamCipherRoundtrip(t *testing.T) { - key := make([]byte, chacha20.KeySize+1) - iv := make([]byte, chacha20.NonceSizeX) + key := common.BlobKeyFromBytes(make([]byte, chacha20.KeySize+1)) + iv := common.BlobIVFromBytes(make([]byte, chacha20.NonceSizeX)) data := []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06} buf := bytes.NewBuffer(nil) diff --git a/pkg/internal/utilities/cipherfactory/generator.go b/pkg/internal/utilities/cipherfactory/generator.go index 6c07cb4..9f9c3da 100644 --- a/pkg/internal/utilities/cipherfactory/generator.go +++ b/pkg/internal/utilities/cipherfactory/generator.go @@ -43,10 +43,10 @@ type keyGenerator struct { func (g keyGenerator) Write(b []byte) (int, error) { return g.h.Write(b) } func (g keyGenerator) Generate() common.BlobKey { - return append( + return common.BlobKeyFromBytes(append( []byte{reservedByteForKeyType}, g.h.Sum(nil)[:chacha20.KeySize]..., - ) + )) } type IVGenerator interface { @@ -61,7 +61,7 @@ type ivGenerator struct { func (g ivGenerator) Write(b []byte) (int, error) { return g.h.Write(b) } func (g ivGenerator) Generate() common.BlobIV { - return g.h.Sum(nil)[:chacha20.NonceSizeX] + return common.BlobIVFromBytes(g.h.Sum(nil)[:chacha20.NonceSizeX]) } func NewKeyGenerator(t common.BlobType) KeyGenerator { @@ -80,7 +80,7 @@ func NewIVGenerator(t common.BlobType) IVGenerator { var defaultXChaCha20IV = func() common.BlobIV { h := sha256.New() h.Write([]byte{preambleHashDefaultIV, reservedByteForKeyType}) - return h.Sum(nil)[:chacha20.NonceSizeX] + return common.BlobIVFromBytes(h.Sum(nil)[:chacha20.NonceSizeX]) }() func DefaultIV(k common.BlobKey) common.BlobIV { diff --git a/pkg/internal/utilities/cipherfactory/generator_test.go b/pkg/internal/utilities/cipherfactory/generator_test.go index 316d1ef..3cb5710 100644 --- a/pkg/internal/utilities/cipherfactory/generator_test.go +++ b/pkg/internal/utilities/cipherfactory/generator_test.go @@ -55,9 +55,12 @@ func TestGenerator(t *testing.T) { // then the generation of key and iv for the same input dataset would // be using same hashed dataset which may be exploitable since IV // is made public - require.NotEqual(t, key[1:1+8], iv[:8]) - require.NotEqual(t, key[1:1+8], DefaultIV(key)[:8]) - require.NotEqual(t, iv[:8], DefaultIV(key)[:8]) + keyBytes := key.Bytes() + ivBytes := iv.Bytes() + defIvBytes := DefaultIV(key).Bytes() + require.NotEqual(t, keyBytes[1:1+8], ivBytes[:8]) + require.NotEqual(t, keyBytes[1:1+8], defIvBytes[:8]) + require.NotEqual(t, ivBytes[:8], defIvBytes[:8]) // Note: once other key types are introduced, we should also check // that for different key types there are different hashes diff --git a/pkg/structure/cinodefs.go b/pkg/structure/cinodefs.go index 4c863a7..02005e3 100644 --- a/pkg/structure/cinodefs.go +++ b/pkg/structure/cinodefs.go @@ -39,7 +39,7 @@ func (d *CinodeFS) OpenContent(ctx context.Context, ep *protobuf.Entrypoint) (io return d.BE.Open( ctx, common.BlobName(ep.BlobName), - common.BlobKey(ep.GetKeyInfo().GetKey()), + common.BlobKeyFromBytes(ep.GetKeyInfo().GetKey()), ) } diff --git a/pkg/structure/directory.go b/pkg/structure/directory.go index 344c3ca..24ca29c 100644 --- a/pkg/structure/directory.go +++ b/pkg/structure/directory.go @@ -122,7 +122,7 @@ func UploadStaticBlob(ctx context.Context, be blenc.BE, r io.Reader, mimeType st // This buffer may then be used to detect the mime type dataHead := newHeadWriter(512) - bn, ki, _, err := be.Create(context.Background(), blobtypes.Static, io.TeeReader(r, &dataHead)) + bn, key, _, err := be.Create(context.Background(), blobtypes.Static, io.TeeReader(r, &dataHead)) if err != nil { log.ErrorContext(ctx, "failed to upload static file", "err", err) return nil, err @@ -137,7 +137,7 @@ func UploadStaticBlob(ctx context.Context, be blenc.BE, r io.Reader, mimeType st return &protobuf.Entrypoint{ BlobName: bn, - KeyInfo: &protobuf.KeyInfo{Key: ki}, + KeyInfo: &protobuf.KeyInfo{Key: key.Bytes()}, MimeType: mimeType, }, nil } @@ -266,14 +266,14 @@ func (s *StaticDir) GenerateEntrypoint(ctx context.Context, be blenc.BE) (*proto return nil, err } - bn, ki, _, err := be.Create(context.Background(), blobtypes.Static, bytes.NewReader(data)) + bn, key, _, err := be.Create(context.Background(), blobtypes.Static, bytes.NewReader(data)) if err != nil { return nil, err } return &protobuf.Entrypoint{ BlobName: bn, - KeyInfo: &protobuf.KeyInfo{Key: ki}, + KeyInfo: &protobuf.KeyInfo{Key: key.Bytes()}, MimeType: CinodeDirMimeType, }, nil } diff --git a/pkg/structure/link.go b/pkg/structure/link.go index 36070c6..69c51f2 100644 --- a/pkg/structure/link.go +++ b/pkg/structure/link.go @@ -51,11 +51,11 @@ func CreateLink(ctx context.Context, be blenc.BE, ep *protobuf.Entrypoint) (*pro return &protobuf.Entrypoint{ BlobName: name, KeyInfo: &protobuf.KeyInfo{ - Key: key, + Key: key.Bytes(), }, }, &protobuf.WriterInfo{ BlobName: name, - Key: key, + Key: key.Bytes(), AuthInfo: authInfo, }, nil } @@ -66,7 +66,13 @@ func UpdateLink(ctx context.Context, be blenc.BE, wi *protobuf.WriterInfo, ep *p return nil, err } - err = be.Update(ctx, wi.BlobName, wi.AuthInfo, wi.Key, bytes.NewReader(epBytes)) + err = be.Update( + ctx, + wi.BlobName, + wi.AuthInfo, + common.BlobKeyFromBytes(wi.Key), + bytes.NewReader(epBytes), + ) if err != nil { return nil, err } @@ -103,7 +109,7 @@ func DereferenceLink( rc, err := be.Open( ctx, common.BlobName(link.BlobName), - common.BlobKey(link.GetKeyInfo().GetKey()), + common.BlobKeyFromBytes(link.GetKeyInfo().GetKey()), ) if err != nil { return nil, err From 08fe797e8b28952ad34d2f4c7b0d1e10d9e7dca9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Tue, 19 Sep 2023 00:10:48 +0200 Subject: [PATCH 04/29] BlobName refactor --- pkg/blenc/datastore.go | 2 +- pkg/blenc/datastore_dynamic_link.go | 9 +++-- pkg/blenc/datastore_dynamic_link_test.go | 12 +++---- pkg/blenc/datastore_static.go | 18 +++++----- pkg/blenc/datastore_static_test.go | 36 +++++++++---------- pkg/blenc/interface_test.go | 12 +++---- pkg/cmd/cinode_web_proxy/root_test.go | 2 +- pkg/common/blob_name.go | 24 +++++++++---- pkg/common/blob_name_test.go | 9 ++++- pkg/datastore/utils_autogen_for_test.go | 25 ++++++------- pkg/datastore/webinterface.go | 6 ++-- pkg/internal/blobtypes/dynamiclink/public.go | 8 ++--- .../blobtypes/dynamiclink/publisher.go | 2 +- .../blobtypes/dynamiclink/vectors_test.go | 12 +++++-- pkg/protobuf/protobuf.go | 23 +++++++++++- pkg/structure/cinodefs.go | 12 +++---- pkg/structure/directory.go | 7 ++-- pkg/structure/link.go | 23 ++++++------ 18 files changed, 146 insertions(+), 96 deletions(-) diff --git a/pkg/blenc/datastore.go b/pkg/blenc/datastore.go index 1f47caa..78cc2cf 100644 --- a/pkg/blenc/datastore.go +++ b/pkg/blenc/datastore.go @@ -76,7 +76,7 @@ func (be *beDatastore) Create( case blobtypes.DynamicLink: return be.createDynamicLink(ctx, r) } - return nil, common.BlobKey{}, nil, blobtypes.ErrUnknownBlobType + return common.BlobName{}, common.BlobKey{}, nil, blobtypes.ErrUnknownBlobType } func (be *beDatastore) Update(ctx context.Context, name common.BlobName, authInfo AuthInfo, key common.BlobKey, r io.Reader) error { diff --git a/pkg/blenc/datastore_dynamic_link.go b/pkg/blenc/datastore_dynamic_link.go index 6c42dc1..b78de2f 100644 --- a/pkg/blenc/datastore_dynamic_link.go +++ b/pkg/blenc/datastore_dynamic_link.go @@ -17,7 +17,6 @@ limitations under the License. package blenc import ( - "bytes" "context" "errors" "fmt" @@ -85,19 +84,19 @@ func (be *beDatastore) createDynamicLink( dl, err := dynamiclink.Create(be.rand) if err != nil { - return nil, common.BlobKey{}, nil, err + return common.BlobName{}, common.BlobKey{}, nil, err } pr, encryptionKey, err := dl.UpdateLinkData(r, version) if err != nil { - return nil, common.BlobKey{}, nil, err + return common.BlobName{}, common.BlobKey{}, nil, err } // Send update packet bn := dl.BlobName() err = be.ds.Update(ctx, bn, pr.GetPublicDataReader()) if err != nil { - return nil, common.BlobKey{}, nil, err + return common.BlobName{}, common.BlobKey{}, nil, err } return bn, @@ -129,7 +128,7 @@ func (be *beDatastore) updateDynamicLink( if !encryptionKey.Equal(key) { return ErrDynamicLinkUpdateFailedWrongKey } - if !bytes.Equal(name, dl.BlobName()) { + if !name.Equal(dl.BlobName()) { return ErrDynamicLinkUpdateFailedWrongName } diff --git a/pkg/blenc/datastore_dynamic_link_test.go b/pkg/blenc/datastore_dynamic_link_test.go index 8e1b12f..c1942d5 100644 --- a/pkg/blenc/datastore_dynamic_link_test.go +++ b/pkg/blenc/datastore_dynamic_link_test.go @@ -124,9 +124,9 @@ func TestDynamicLinkErrors(t *testing.T) { bn, key, ai, err := be.Create(context.Background(), blobtypes.DynamicLink, bytes.NewReader(nil)) require.ErrorIs(t, err, injectedErr) - require.Nil(t, bn) - require.Equal(t, common.BlobKey{}, key) - require.Nil(t, ai) + require.Empty(t, bn) + require.Empty(t, key) + require.Empty(t, ai) be.(*beDatastore).rand = rand.Reader @@ -139,9 +139,9 @@ func TestDynamicLinkErrors(t *testing.T) { bn, key, ai, err := be.Create(context.Background(), blobtypes.DynamicLink, bytes.NewReader(nil)) require.ErrorIs(t, err, injectedErr) - require.Nil(t, bn) - require.Equal(t, common.BlobKey{}, key) - require.Nil(t, ai) + require.Empty(t, bn) + require.Empty(t, key) + require.Empty(t, ai) dsw.updateFn = nil }) diff --git a/pkg/blenc/datastore_static.go b/pkg/blenc/datastore_static.go index 4ebac95..4dcc052 100644 --- a/pkg/blenc/datastore_static.go +++ b/pkg/blenc/datastore_static.go @@ -74,20 +74,20 @@ func (be *beDatastore) createStatic( ) { tempWriteBufferPlain, err := be.newSecureFifo() if err != nil { - return nil, common.BlobKey{}, nil, err + return common.BlobName{}, common.BlobKey{}, nil, err } defer tempWriteBufferPlain.Close() tempWriteBufferEncrypted, err := be.newSecureFifo() if err != nil { - return nil, common.BlobKey{}, nil, err + return common.BlobName{}, common.BlobKey{}, nil, err } defer tempWriteBufferEncrypted.Close() keyGenerator := cipherfactory.NewKeyGenerator(blobtypes.Static) _, err = io.Copy(tempWriteBufferPlain, io.TeeReader(r, keyGenerator)) if err != nil { - return nil, common.BlobKey{}, nil, err + return common.BlobName{}, common.BlobKey{}, nil, err } key := keyGenerator.Generate() @@ -95,7 +95,7 @@ func (be *beDatastore) createStatic( rClone, err := tempWriteBufferPlain.Done() // rClone will allow re-reading the source data if err != nil { - return nil, common.BlobKey{}, nil, err + return common.BlobName{}, common.BlobKey{}, nil, err } defer rClone.Close() @@ -109,30 +109,30 @@ func (be *beDatastore) createStatic( ), ) if err != nil { - return nil, common.BlobKey{}, nil, err + return common.BlobName{}, common.BlobKey{}, nil, err } _, err = io.Copy(encWriter, rClone) if err != nil { - return nil, common.BlobKey{}, nil, err + return common.BlobName{}, common.BlobKey{}, nil, err } encReader, err := tempWriteBufferEncrypted.Done() if err != nil { - return nil, common.BlobKey{}, nil, err + return common.BlobName{}, common.BlobKey{}, nil, err } defer encReader.Close() // Generate blob name from the encrypted data name, err := common.BlobNameFromHashAndType(blobNameHasher.Sum(nil), blobtypes.Static) if err != nil { - return nil, common.BlobKey{}, nil, err + return common.BlobName{}, common.BlobKey{}, nil, err } // Send encrypted blob into the datastore err = be.ds.Update(ctx, name, encReader) if err != nil { - return nil, common.BlobKey{}, nil, err + return common.BlobName{}, common.BlobKey{}, nil, err } return name, key, nil, nil diff --git a/pkg/blenc/datastore_static_test.go b/pkg/blenc/datastore_static_test.go index 82747ef..b521e84 100644 --- a/pkg/blenc/datastore_static_test.go +++ b/pkg/blenc/datastore_static_test.go @@ -97,9 +97,9 @@ func TestStaticErrorTruncatedDatastore(t *testing.T) { bn, key, ai, err := be.Create(context.Background(), blobtypes.Static, bytes.NewReader(nil)) require.ErrorIs(t, err, injectedErr) - require.Nil(t, bn) - require.Equal(t, common.BlobKey{}, key) - require.Nil(t, ai) + require.Empty(t, bn) + require.Empty(t, key) + require.Empty(t, ai) }) t.Run("second securefifo", func(t *testing.T) { @@ -127,9 +127,9 @@ func TestStaticErrorTruncatedDatastore(t *testing.T) { bn, key, ai, err := be.Create(context.Background(), blobtypes.Static, bytes.NewReader(nil)) require.ErrorIs(t, err, injectedErr) - require.Nil(t, bn) - require.Equal(t, common.BlobKey{}, key) - require.Nil(t, ai) + require.Empty(t, bn) + require.Empty(t, key) + require.Empty(t, ai) require.True(t, firstSecureFifoCreated) require.True(t, firstSecureFifoClosed) }) @@ -165,9 +165,9 @@ func TestStaticErrorTruncatedDatastore(t *testing.T) { bn, key, ai, err := be.Create(context.Background(), blobtypes.Static, bytes.NewReader(nil)) require.ErrorIs(t, err, injectedErr) - require.Nil(t, bn) - require.Equal(t, common.BlobKey{}, key) - require.Nil(t, ai) + require.Empty(t, bn) + require.Empty(t, key) + require.Empty(t, ai) require.Equal(t, 2, secureFifosCreated) require.Equal(t, secureFifosCreated, secureFifosClosed) }) @@ -204,9 +204,9 @@ func TestStaticErrorTruncatedDatastore(t *testing.T) { bn, key, ai, err := be.Create(context.Background(), blobtypes.Static, bytes.NewReader([]byte("Hello world"))) require.ErrorIs(t, err, injectedErr) - require.Nil(t, bn) - require.Equal(t, common.BlobKey{}, key) - require.Nil(t, ai) + require.Empty(t, bn) + require.Empty(t, key) + require.Empty(t, ai) require.Equal(t, 2, secureFifosCreated) require.Equal(t, secureFifosCreated, secureFifosClosed) }) @@ -236,9 +236,9 @@ func TestStaticErrorTruncatedDatastore(t *testing.T) { bn, key, ai, err := be.Create(context.Background(), blobtypes.Static, iotest.ErrReader(injectedErr)) require.ErrorIs(t, err, injectedErr) - require.Nil(t, bn) - require.Equal(t, common.BlobKey{}, key) - require.Nil(t, ai) + require.Empty(t, bn) + require.Empty(t, key) + require.Empty(t, ai) require.Equal(t, 2, secureFifosCreated) require.Equal(t, secureFifosCreated, secureFifosClosed) }) @@ -271,9 +271,9 @@ func TestStaticErrorTruncatedDatastore(t *testing.T) { bn, key, ai, err := be.Create(context.Background(), blobtypes.Static, bytes.NewReader(nil)) require.ErrorIs(t, err, injectedErr) - require.Nil(t, bn) - require.Equal(t, common.BlobKey{}, key) - require.Nil(t, ai) + require.Empty(t, bn) + require.Empty(t, key) + require.Empty(t, ai) require.Equal(t, 2, secureFifosCreated) require.Equal(t, secureFifosCreated, secureFifosClosed) diff --git a/pkg/blenc/interface_test.go b/pkg/blenc/interface_test.go index 39eaf41..1f3f174 100644 --- a/pkg/blenc/interface_test.go +++ b/pkg/blenc/interface_test.go @@ -243,9 +243,9 @@ func (s *BlencTestSuite) TestDynamicLinkSuccessPath() { bn, key, ai, err := s.be.Create(context.Background(), blobtypes.DynamicLink, iotest.ErrReader(injectedErr)) s.Require().ErrorIs(err, injectedErr) - s.Require().Nil(bn) - s.Require().Equal(common.BlobKey{}, key) - s.Require().Nil(ai) + s.Require().Empty(bn) + s.Require().Empty(key) + s.Require().Empty(ai) }) } @@ -256,9 +256,9 @@ func (s *BlencTestSuite) TestInvalidBlobTypes() { s.Run("must fail to create blob of invalid type", func() { bn, key, ai, err := s.be.Create(context.Background(), blobtypes.Invalid, bytes.NewReader(nil)) s.Require().ErrorIs(err, blobtypes.ErrUnknownBlobType) - s.Require().Nil(bn) - s.Require().Equal(common.BlobKey{}, key) - s.Require().Nil(ai) + s.Require().Empty(bn) + s.Require().Empty(key) + s.Require().Empty(ai) }) s.Run("must fail to open blob of invalid type", func() { diff --git a/pkg/cmd/cinode_web_proxy/root_test.go b/pkg/cmd/cinode_web_proxy/root_test.go index 98aef04..a6a9fe7 100644 --- a/pkg/cmd/cinode_web_proxy/root_test.go +++ b/pkg/cmd/cinode_web_proxy/root_test.go @@ -115,7 +115,7 @@ func TestWebProxyHandlerInvalidEntrypoint(t *testing.T) { datastore.InMemory(), []datastore.DS{}, &protobuf.Entrypoint{ - BlobName: n, + BlobName: n.Bytes(), MimeType: structure.CinodeDirMimeType, KeyInfo: &protobuf.KeyInfo{ Key: cipherfactory.NewKeyGenerator(blobtypes.Static).Generate().Bytes(), diff --git a/pkg/common/blob_name.go b/pkg/common/blob_name.go index c6d57b0..ea7cddf 100644 --- a/pkg/common/blob_name.go +++ b/pkg/common/blob_name.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ limitations under the License. package common import ( + "crypto/subtle" "errors" base58 "github.com/jbenet/go-base58" @@ -38,7 +39,7 @@ type BlobName []byte // and given blob type func BlobNameFromHashAndType(hash []byte, t BlobType) (BlobName, error) { if len(hash) == 0 || len(hash) > 0x7E { - return nil, ErrInvalidBlobName + return BlobName{}, ErrInvalidBlobName } ret := make([]byte, len(hash)+1) @@ -56,11 +57,14 @@ func BlobNameFromHashAndType(hash []byte, t BlobType) (BlobName, error) { // BlobNameFromString decodes base58-encoded string into blob name func BlobNameFromString(s string) (BlobName, error) { - decoded := base58.Decode(s) - if len(decoded) == 0 || len(decoded) > 0x7F { - return nil, ErrInvalidBlobName + return BlobNameFromBytes(base58.Decode(s)) +} + +func BlobNameFromBytes(n []byte) (BlobName, error) { + if len(n) == 0 || len(n) > 0x7F { + return BlobName{}, ErrInvalidBlobName } - return BlobName(decoded), nil + return BlobName(copyBytes(n)), nil } // Returns base58-encoded blob name @@ -81,3 +85,11 @@ func (b BlobName) Type() BlobType { } return BlobType{t: ret} } + +func (b BlobName) Bytes() []byte { + return copyBytes(b) +} + +func (b BlobName) Equal(b2 BlobName) bool { + return subtle.ConstantTimeCompare(b, b2) == 1 +} diff --git a/pkg/common/blob_name_test.go b/pkg/common/blob_name_test.go index 28d1550..56682e2 100644 --- a/pkg/common/blob_name_test.go +++ b/pkg/common/blob_name_test.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -51,6 +51,13 @@ func TestBlobName(t *testing.T) { bn2, err := BlobNameFromString(s) require.NoError(t, err) require.Equal(t, bn, bn2) + require.True(t, bn.Equal(bn2)) + + b := bn.Bytes() + bn3, err := BlobNameFromBytes(b) + require.NoError(t, err) + require.Equal(t, bn, bn3) + require.True(t, bn.Equal(bn3)) }) } } diff --git a/pkg/datastore/utils_autogen_for_test.go b/pkg/datastore/utils_autogen_for_test.go index 91a5491..2e928e1 100644 --- a/pkg/datastore/utils_autogen_for_test.go +++ b/pkg/datastore/utils_autogen_for_test.go @@ -26,6 +26,7 @@ import ( "github.com/cinode/go/pkg/blobtypes" "github.com/cinode/go/pkg/common" "github.com/cinode/go/pkg/internal/blobtypes/dynamiclink" + "github.com/cinode/go/pkg/utilities/golang" "github.com/jbenet/go-base58" ) @@ -36,32 +37,32 @@ var testBlobs = []struct { }{ // Static blobs { - common.BlobName(base58.Decode("KDc2ijtWc9mGxb5hP29YSBgkMLH8wCWnVimpvP3M6jdAk")), + golang.Must(common.BlobNameFromString("KDc2ijtWc9mGxb5hP29YSBgkMLH8wCWnVimpvP3M6jdAk")), base58.Decode("3A836b"), base58.Decode("3A836b"), }, { - common.BlobName(base58.Decode("BG8WaXMAckEfbCuoiHpx2oMAS4zAaPqAqrgf5Q3YNzmHx")), + golang.Must(common.BlobNameFromString("BG8WaXMAckEfbCuoiHpx2oMAS4zAaPqAqrgf5Q3YNzmHx")), base58.Decode("AXG4Ffv"), base58.Decode("AXG4Ffv"), }, { - common.BlobName(base58.Decode("2GLoj4Bk7SvjQngCT85gxWRu2DXCCjs9XWKsSpM85Wq3Ve")), + golang.Must(common.BlobNameFromString("2GLoj4Bk7SvjQngCT85gxWRu2DXCCjs9XWKsSpM85Wq3Ve")), base58.Decode(""), base58.Decode(""), }, { - common.BlobName(base58.Decode("251SEdnHjwyvUqX1EZnuKruta4yHMkTDed7LGoi3nUJwhx")), + golang.Must(common.BlobNameFromString("251SEdnHjwyvUqX1EZnuKruta4yHMkTDed7LGoi3nUJwhx")), base58.Decode("1DhLfjA9ij9QFBh7J8ysnN3uvGcsNQa7vaxKEwbYEMSEXuZbgyCtUAn5M4QxmgLVnCJ6cARY5Ry2EJVXxn48D837xGxRp1M2rRnz9BHVGw2sc9Ee1DkLmsurGoKX1Evt2iuMhNQyNGh2CrsHWxdGTvZVhpHShmKRziHZEDybK4ZaJh9RvTEngYQkeHAtC3J3TW6dbpaNWBNLD6YdU5xPcaE3AUPMnk4CM1dD8XMBRQekZguNJHNZwNQCXRQodVyGLVRzi1dkTG2odnrcbZ4i3oNxyJyz"), base58.Decode("4wNoVjVdtJ5FKtD3ZmHW4bvTiWgZFmwmps9JEJxDdinXscjMWjjeTQo2Hzwkg6GnFp1kmNoSZR9d5hXnG4qHi6mx2KqM7SVJ"), }, { - common.BlobName(base58.Decode("27vP1JG4VJNZvQJ4Zfhy3H5xKugurbh89B7rKTcStM9guB")), + golang.Must(common.BlobNameFromString("27vP1JG4VJNZvQJ4Zfhy3H5xKugurbh89B7rKTcStM9guB")), base58.Decode("1eE2wp1836WtQmEbjdavggJvFPU7dZbQQH5EBS2LwBL2rYjArM9mjvWCrAbpZDkLFx7dQ5FyejnHD1EbwofDDLa1zNmN94qws1UfhNM4KCBT4oijCfPbJHobp7h5tcZQwMZy1gA3jTQBRvem2ioNuSFwqKRwbVJs9S21QFB86XuuUggNmj6sfAsDKwvE4M5EQxSkDft3CFiUX6XUMgCJUAreBRoT32wz7ncNbFaETMscFTTjFUYYiUFuv6fQESbfDCV3rfcSmxSLbLqm2u2Pd83cnzqfH"), base58.Decode("8Ya88xk8C7tnYXAKJL9t1bCoBUur9dLr44SwhchsBh7UQb7TmZihVpffndCxLmhH9YjMrQj442YhiW3Hr2bBUR4vCcn6VdJLK"), }, { - common.BlobName(base58.Decode("e3T1HcdDLc73NHed2SFu5XHUQx5KDwgdAYTMmmEk2Ekqm")), + golang.Must(common.BlobNameFromString("e3T1HcdDLc73NHed2SFu5XHUQx5KDwgdAYTMmmEk2Ekqm")), base58.Decode("1yULPpEx3gjpKNBLCEzb2oj2xRcdGfr88CztgfYfEipBGiJCijqWBEEhXTaReU6CBcbt61h2DeGoZhgAfTiEwppGkJWCJrtmkSiLiib8UhupERptC3U2j6BKDg8PLwHq113WKJWM4tr2c3WxTXTSosjk7fBhuz3GJgqdYLecBfnKMGUqw8XkBf2Lth2REAw4ccZmmYn21x1W1tFdVCe4cAzAEqc5adJC3j3prPsYvL8QSqBZE5nQcnvfGekTUqn7HDZbZvqFN3TKc8HSVK9YUQ"), base58.Decode("MpaLZEfQpasGN1khuvpTC6CFnJucjVmzRfZwaxJkti1uQAetXnvDL8PmrFHZkr7XX1GtKaQqB2P6M2KZjCYCTfxMZi"), }, @@ -73,17 +74,17 @@ var dynamicLinkPropagationData = []struct { expected []byte }{ { - common.BlobName(base58.Decode("GUnL66Lyv2Qs4baxPhy59kF4dsB9HWakvTjMBjNGFLT6g")), + golang.Must(common.BlobNameFromString("GUnL66Lyv2Qs4baxPhy59kF4dsB9HWakvTjMBjNGFLT6g")), base58.Decode("17Jk3QJMCABypJWuAivYwMi43gN1KxPVy3qg1e4HYQFe8BCQPVm6GX8auaFjXhwZZQhxaHjDirXH6Ze59irpWSkBicnqigPcd6j5H9AjnPHTHRKhyLSSX5kqkVRiwSRvTojGvx6oeMqj2hyhK9LxStjtYVW7WKxoCwATgQbkUWRszH2Eff3bHND8RbknhfZDSvSmXxSR8h6tMTErcV8dGyPYUysdV6Gd9bEK8bjRs6NxhCLpQ55dvZcwEi6i7rqo2WQWhY7HMMhmKhggvLXcReaUMTByq"), base58.Decode("PnB1W5tcQdkzYrnvE8Z1BAsBgv9kVgdeZMp78WYxnJJKi2RDPHgx9VvzYZ1hzGhVxBetGfuxwdstH8E9oNiUQ6JDNPWYZAXE7"), }, { - common.BlobName(base58.Decode("GUnL66Lyv2Qs4baxPhy59kF4dsB9HWakvTjMBjNGFLT6g")), + golang.Must(common.BlobNameFromString("GUnL66Lyv2Qs4baxPhy59kF4dsB9HWakvTjMBjNGFLT6g")), base58.Decode("17Jk3QJMCABypJWuAivYwMi43gN1KxPVy3qg1e4HYQFe8BCQPVm6GX8cxHJfhdYkZjq51cCNcGKTXirXxYcaGA5XdykeezU9P6jE72kmmpLNthhndMuE9oz7p725mWqPYMbMiw4Qp54oiRWdxEvh3yKRvjRA7MFK9ZJKGY1evFGbqsaMAE715aRYvP3yNjE7FaNwkKbAn1xJm4ojF4qjtaNN5zxHRgQfdZYLgybbsYJ3TJUNMxxNPkqu2CsiieeKJpJce8U5g3HAP6jAKSiXMBcmBfGm8"), base58.Decode("mgbcvX3FFDqwigwuybL2misVJSLjXzZs9bumic8rFSHCD9nMqbmsTxNWnRpoVn3E2GKQaFcUdUzhMax1oiq5X9abrKYqXYMtN"), }, { - common.BlobName(base58.Decode("GUnL66Lyv2Qs4baxPhy59kF4dsB9HWakvTjMBjNGFLT6g")), + golang.Must(common.BlobNameFromString("GUnL66Lyv2Qs4baxPhy59kF4dsB9HWakvTjMBjNGFLT6g")), base58.Decode("17Jk3QJMCABypJWuAivYwMi43gN1KxPVy3qg1e4HYQFe8BCQPVm6GX8bbhTds8inbAV58TPXDmM19FuZEFGP1B3w9gBPoTfVQUfrhmB2A4uBrcKSxFBMNT8djhviFuunpME39ZSEZp3KS4w1jms7gKnoG237vs4vnNn4uRVF6pj5oorff4VxECGVektdbkiU2BcAUQUHbkqkcw3f3sX5Rtw5Ckv5mBzaa4zqUtLiK7eYp8Wqc5Au7mzTuXvPDpWbX85hz7EnDsuHQEoZAeFCFeWdzZSgS"), base58.Decode("WZpzxEiTLyv42JwAfYCTo7TckS1bLY6XmuoJWoqz8BVzYNqUSvDf58KJR6tjuEegLRYCkiprPskdP7PMFP6wazLxed8JEPAsC"), }, @@ -92,14 +93,14 @@ var dynamicLinkPropagationData = []struct { func TestDatasetGeneration(t *testing.T) { t.SkipNow() - dumpBlob := func(name, content []byte, expected []byte) { + dumpBlob := func(name common.BlobName, content []byte, expected []byte) { fmt.Printf(""+ " {\n"+ - " common.BlobName(base58.Decode(\"%s\")),\n"+ + " golang.Must(common.BlobNameFromString(\"%s\")),\n"+ " base58.Decode(\"%s\"),\n"+ " base58.Decode(\"%s\"),\n"+ " },\n", - base58.Encode(name), + name.String(), base58.Encode(content), base58.Encode(expected), ) diff --git a/pkg/datastore/webinterface.go b/pkg/datastore/webinterface.go index 60e03c0..a4f0eac 100644 --- a/pkg/datastore/webinterface.go +++ b/pkg/datastore/webinterface.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -79,12 +79,12 @@ func (i *webInterface) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (i *webInterface) getName(w http.ResponseWriter, r *http.Request) (common.BlobName, error) { // Don't allow url queries and require path to start with '/' if r.URL.Path[0] != '/' || r.URL.RawQuery != "" { - return nil, common.ErrInvalidBlobName + return common.BlobName{}, common.ErrInvalidBlobName } bn, err := common.BlobNameFromString(r.URL.Path[1:]) if err != nil { - return nil, err + return common.BlobName{}, err } return bn, nil diff --git a/pkg/internal/blobtypes/dynamiclink/public.go b/pkg/internal/blobtypes/dynamiclink/public.go index 0002799..f5f3f1a 100644 --- a/pkg/internal/blobtypes/dynamiclink/public.go +++ b/pkg/internal/blobtypes/dynamiclink/public.go @@ -119,7 +119,7 @@ func FromPublicData(name common.BlobName, r io.Reader) (*PublicReader, error) { return nil, err } - if !bytes.Equal(dl.BlobName(), name) { + if !dl.BlobName().Equal(name) { return nil, ErrInvalidDynamicLinkDataBlobName } @@ -214,7 +214,7 @@ func (d *PublicReader) toSignDataHasherPrefilled() hash.Hash { h := sha256.New() storeByte(h, signatureForLinkData) - storeDynamicSizeBuff(h, d.BlobName()) + storeDynamicSizeBuff(h, d.BlobName().Bytes()) return h } @@ -239,7 +239,7 @@ func (d *PublicReader) GreaterThan(d2 *PublicReader) bool { func (d *PublicReader) ivGeneratorPrefilled() cipherfactory.IVGenerator { ivGenerator := cipherfactory.NewIVGenerator(blobtypes.DynamicLink) - storeDynamicSizeBuff(ivGenerator, d.BlobName()) + storeDynamicSizeBuff(ivGenerator, d.BlobName().Bytes()) storeUint64(ivGenerator, d.contentVersion) return ivGenerator @@ -257,7 +257,7 @@ func (d *PublicReader) validateKeyInLinkData(key common.BlobKey, r io.Reader) er dataSeed := append( []byte{signatureForEncryptionKeyGeneration}, - d.BlobName()..., + d.BlobName().Bytes()..., ) // Key validation block contains the signature of data seed diff --git a/pkg/internal/blobtypes/dynamiclink/publisher.go b/pkg/internal/blobtypes/dynamiclink/publisher.go index 63f5c0e..6d82373 100644 --- a/pkg/internal/blobtypes/dynamiclink/publisher.go +++ b/pkg/internal/blobtypes/dynamiclink/publisher.go @@ -110,7 +110,7 @@ func (dl *Publisher) AuthInfo() []byte { func (dl *Publisher) calculateEncryptionKey() (common.BlobKey, []byte) { dataSeed := append( []byte{signatureForEncryptionKeyGeneration}, - dl.BlobName()..., + dl.BlobName().Bytes()..., ) signature := ed25519.Sign(dl.privKey, dataSeed) diff --git a/pkg/internal/blobtypes/dynamiclink/vectors_test.go b/pkg/internal/blobtypes/dynamiclink/vectors_test.go index 4303546..8a41646 100644 --- a/pkg/internal/blobtypes/dynamiclink/vectors_test.go +++ b/pkg/internal/blobtypes/dynamiclink/vectors_test.go @@ -67,8 +67,12 @@ func TestVectors(t *testing.T) { t.Run(testCase.Name, func(t *testing.T) { t.Run("validate public scope", func(t *testing.T) { err := func() error { + bn, err := common.BlobNameFromBytes(testCase.BlobName) + if err != nil { + return err + } pr, err := FromPublicData( - common.BlobName(testCase.BlobName), + bn, bytes.NewReader(testCase.UpdateDataset), ) if err != nil { @@ -94,9 +98,13 @@ func TestVectors(t *testing.T) { t.Run("validate private scope", func(t *testing.T) { err := func() error { + bn, err := common.BlobNameFromBytes(testCase.BlobName) + if err != nil { + return err + } pr, err := FromPublicData( - common.BlobName(testCase.BlobName), + bn, bytes.NewReader(testCase.UpdateDataset), ) if err != nil { diff --git a/pkg/protobuf/protobuf.go b/pkg/protobuf/protobuf.go index 8dae6e1..b585a8d 100644 --- a/pkg/protobuf/protobuf.go +++ b/pkg/protobuf/protobuf.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ import ( "errors" "time" + "github.com/cinode/go/pkg/common" "google.golang.org/protobuf/proto" ) @@ -71,3 +72,23 @@ func (ep *Entrypoint) Validate(currentTime time.Time) error { return nil } + +func (ep *Entrypoint) ValidateAndParse(currentTime time.Time) ( + common.BlobName, + common.BlobKey, + error, +) { + err := ep.Validate(currentTime) + if err != nil { + return common.BlobName{}, common.BlobKey{}, err + } + + bn, err := common.BlobNameFromBytes(ep.BlobName) + if err != nil { + return common.BlobName{}, common.BlobKey{}, err + } + + key := common.BlobKeyFromBytes(ep.GetKeyInfo().GetKey()) + + return bn, key, nil +} diff --git a/pkg/structure/cinodefs.go b/pkg/structure/cinodefs.go index 02005e3..360edbe 100644 --- a/pkg/structure/cinodefs.go +++ b/pkg/structure/cinodefs.go @@ -23,7 +23,6 @@ import ( "time" "github.com/cinode/go/pkg/blenc" - "github.com/cinode/go/pkg/common" "github.com/cinode/go/pkg/protobuf" "google.golang.org/protobuf/proto" ) @@ -36,11 +35,12 @@ type CinodeFS struct { } func (d *CinodeFS) OpenContent(ctx context.Context, ep *protobuf.Entrypoint) (io.ReadCloser, error) { - return d.BE.Open( - ctx, - common.BlobName(ep.BlobName), - common.BlobKeyFromBytes(ep.GetKeyInfo().GetKey()), - ) + bn, key, err := ep.ValidateAndParse(time.Now()) + if err != nil { + return nil, err + } + + return d.BE.Open(ctx, bn, key) } func (d *CinodeFS) FindEntrypoint(ctx context.Context, path string) (*protobuf.Entrypoint, error) { diff --git a/pkg/structure/directory.go b/pkg/structure/directory.go index 24ca29c..0e884d4 100644 --- a/pkg/structure/directory.go +++ b/pkg/structure/directory.go @@ -136,7 +136,7 @@ func UploadStaticBlob(ctx context.Context, be blenc.BE, r io.Reader, mimeType st } return &protobuf.Entrypoint{ - BlobName: bn, + BlobName: bn.Bytes(), KeyInfo: &protobuf.KeyInfo{Key: key.Bytes()}, MimeType: mimeType, }, nil @@ -190,9 +190,10 @@ func (d *dirCompiler) compileDir(ctx context.Context, p string) (*protobuf.Entry return nil, fmt.Errorf("can not serialize directory %v: %w", p, err) } + bn, _ := common.BlobNameFromBytes(ep.BlobName) d.log.DebugContext(ctx, "directory uploaded successfully", "path", p, - "blobName", common.BlobName(ep.BlobName).String(), + "blobName", bn.String(), ) return ep, nil } @@ -272,7 +273,7 @@ func (s *StaticDir) GenerateEntrypoint(ctx context.Context, be blenc.BE) (*proto } return &protobuf.Entrypoint{ - BlobName: bn, + BlobName: bn.Bytes(), KeyInfo: &protobuf.KeyInfo{Key: key.Bytes()}, MimeType: CinodeDirMimeType, }, nil diff --git a/pkg/structure/link.go b/pkg/structure/link.go index 69c51f2..b0a69b7 100644 --- a/pkg/structure/link.go +++ b/pkg/structure/link.go @@ -49,12 +49,12 @@ func CreateLink(ctx context.Context, be blenc.BE, ep *protobuf.Entrypoint) (*pro } return &protobuf.Entrypoint{ - BlobName: name, + BlobName: name.Bytes(), KeyInfo: &protobuf.KeyInfo{ Key: key.Bytes(), }, }, &protobuf.WriterInfo{ - BlobName: name, + BlobName: name.Bytes(), Key: key.Bytes(), AuthInfo: authInfo, }, nil @@ -66,9 +66,14 @@ func UpdateLink(ctx context.Context, be blenc.BE, wi *protobuf.WriterInfo, ep *p return nil, err } + bn, err := common.BlobNameFromBytes(wi.BlobName) + if err != nil { + return nil, err + } + err = be.Update( ctx, - wi.BlobName, + bn, wi.AuthInfo, common.BlobKeyFromBytes(wi.Key), bytes.NewReader(epBytes), @@ -95,22 +100,18 @@ func DereferenceLink( *protobuf.Entrypoint, error, ) { - err := link.Validate(currentTime) + bn, key, err := link.ValidateAndParse(currentTime) if err != nil { return nil, err } - for common.BlobName(link.BlobName).Type() == blobtypes.DynamicLink { + for bn.Type() == blobtypes.DynamicLink { if maxRedirects == 0 { return nil, ErrMaxRedirectsReached } maxRedirects-- - rc, err := be.Open( - ctx, - common.BlobName(link.BlobName), - common.BlobKeyFromBytes(link.GetKeyInfo().GetKey()), - ) + rc, err := be.Open(ctx, bn, key) if err != nil { return nil, err } @@ -127,7 +128,7 @@ func DereferenceLink( return nil, err } - err = link.Validate(time.Now()) + bn, key, err = link.ValidateAndParse(time.Now()) if err != nil { return nil, err } From 2075eb3b288fcff39244fa5d556b729fd2569ae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Thu, 21 Sep 2023 05:41:16 +0200 Subject: [PATCH 05/29] Make protobuf definitions internal The goal is to create proper API on top of protobuf instead ensuring that we preserve additional internal consistency. --- pkg/structure/cinodefs.go | 2 +- pkg/structure/directory.go | 2 +- pkg/{ => structure/internal}/protobuf/protobuf.go | 0 pkg/{ => structure/internal}/protobuf/protobuf.pb.go | 0 pkg/{ => structure/internal}/protobuf/protobuf.proto | 0 pkg/structure/link.go | 2 +- testvectors/testblobs/base.go | 2 +- 7 files changed, 4 insertions(+), 4 deletions(-) rename pkg/{ => structure/internal}/protobuf/protobuf.go (100%) rename pkg/{ => structure/internal}/protobuf/protobuf.pb.go (100%) rename pkg/{ => structure/internal}/protobuf/protobuf.proto (100%) diff --git a/pkg/structure/cinodefs.go b/pkg/structure/cinodefs.go index 360edbe..6e3ba95 100644 --- a/pkg/structure/cinodefs.go +++ b/pkg/structure/cinodefs.go @@ -23,7 +23,7 @@ import ( "time" "github.com/cinode/go/pkg/blenc" - "github.com/cinode/go/pkg/protobuf" + "github.com/cinode/go/pkg/structure/internal/protobuf" "google.golang.org/protobuf/proto" ) diff --git a/pkg/structure/directory.go b/pkg/structure/directory.go index 0e884d4..b127fd1 100644 --- a/pkg/structure/directory.go +++ b/pkg/structure/directory.go @@ -35,7 +35,7 @@ import ( "github.com/cinode/go/pkg/blenc" "github.com/cinode/go/pkg/blobtypes" "github.com/cinode/go/pkg/common" - "github.com/cinode/go/pkg/protobuf" + "github.com/cinode/go/pkg/structure/internal/protobuf" "github.com/cinode/go/pkg/utilities/golang" "golang.org/x/exp/slog" "google.golang.org/protobuf/proto" diff --git a/pkg/protobuf/protobuf.go b/pkg/structure/internal/protobuf/protobuf.go similarity index 100% rename from pkg/protobuf/protobuf.go rename to pkg/structure/internal/protobuf/protobuf.go diff --git a/pkg/protobuf/protobuf.pb.go b/pkg/structure/internal/protobuf/protobuf.pb.go similarity index 100% rename from pkg/protobuf/protobuf.pb.go rename to pkg/structure/internal/protobuf/protobuf.pb.go diff --git a/pkg/protobuf/protobuf.proto b/pkg/structure/internal/protobuf/protobuf.proto similarity index 100% rename from pkg/protobuf/protobuf.proto rename to pkg/structure/internal/protobuf/protobuf.proto diff --git a/pkg/structure/link.go b/pkg/structure/link.go index b0a69b7..92593e5 100644 --- a/pkg/structure/link.go +++ b/pkg/structure/link.go @@ -26,7 +26,7 @@ import ( "github.com/cinode/go/pkg/blenc" "github.com/cinode/go/pkg/blobtypes" "github.com/cinode/go/pkg/common" - "github.com/cinode/go/pkg/protobuf" + "github.com/cinode/go/pkg/structure/internal/protobuf" ) var ( diff --git a/testvectors/testblobs/base.go b/testvectors/testblobs/base.go index cdd730b..df9f74e 100644 --- a/testvectors/testblobs/base.go +++ b/testvectors/testblobs/base.go @@ -23,7 +23,7 @@ import ( "net/http" "net/url" - "github.com/cinode/go/pkg/protobuf" + "github.com/cinode/go/pkg/structure/internal/protobuf" "github.com/jbenet/go-base58" ) From c18f4f2516023a1c13e1592efe6a5cc9f857b2bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Thu, 5 Oct 2023 13:50:45 +0200 Subject: [PATCH 06/29] First working code and basic cinodefs test --- .../blobtypes/dynamiclink/publisher.go | 5 + pkg/structure/graph/cinodefs.go | 531 ++++++++++++++++++ pkg/structure/graph/cinodefs_blackbox_test.go | 111 ++++ pkg/structure/graph/cinodefs_options.go | 84 +++ pkg/structure/graph/context.go | 140 +++++ pkg/structure/graph/dir.go | 157 ++++++ pkg/structure/graph/entrypoint.go | 126 +++++ pkg/structure/graph/headwriter.go | 27 + pkg/structure/graph/link.go | 66 +++ pkg/structure/graph/writerinfo.go | 47 ++ pkg/utilities/golang/assert.go | 7 + 11 files changed, 1301 insertions(+) create mode 100644 pkg/structure/graph/cinodefs.go create mode 100644 pkg/structure/graph/cinodefs_blackbox_test.go create mode 100644 pkg/structure/graph/cinodefs_options.go create mode 100644 pkg/structure/graph/context.go create mode 100644 pkg/structure/graph/dir.go create mode 100644 pkg/structure/graph/entrypoint.go create mode 100644 pkg/structure/graph/headwriter.go create mode 100644 pkg/structure/graph/link.go create mode 100644 pkg/structure/graph/writerinfo.go create mode 100644 pkg/utilities/golang/assert.go diff --git a/pkg/internal/blobtypes/dynamiclink/publisher.go b/pkg/internal/blobtypes/dynamiclink/publisher.go index 6d82373..66ad207 100644 --- a/pkg/internal/blobtypes/dynamiclink/publisher.go +++ b/pkg/internal/blobtypes/dynamiclink/publisher.go @@ -123,6 +123,11 @@ func (dl *Publisher) calculateEncryptionKey() (common.BlobKey, []byte) { return key, signature } +func (dl *Publisher) EncryptionKey() common.BlobKey { + key, _ := dl.calculateEncryptionKey() + return key +} + func (dl *Publisher) UpdateLinkData(r io.Reader, version uint64) (*PublicReader, common.BlobKey, error) { encryptionKey, kvb := dl.calculateEncryptionKey() diff --git a/pkg/structure/graph/cinodefs.go b/pkg/structure/graph/cinodefs.go new file mode 100644 index 0000000..1eb4c55 --- /dev/null +++ b/pkg/structure/graph/cinodefs.go @@ -0,0 +1,531 @@ +package graph + +import ( + "context" + "crypto/rand" + "errors" + "io" + "mime" + "net/http" + "path/filepath" + "sort" + "time" + + "github.com/cinode/go/pkg/blenc" + "github.com/cinode/go/pkg/blobtypes" + "github.com/cinode/go/pkg/internal/blobtypes/dynamiclink" + "github.com/cinode/go/pkg/structure/internal/protobuf" +) + +var ( + ErrInvalidBE = errors.New("invalid BE argument") + ErrCantOpenDir = errors.New("can not open directory") + ErrTooManyRedirects = errors.New("too many link redirects") + ErrCantComputeBlobKey = errors.New("can not compute blob keys") + ErrModifiedDirectory = errors.New("can not get entrypoint for a directory, unsaved content") + ErrCantDeleteRoot = errors.New("can not delete root object") + ErrNotADirectory = errors.New("entry is not a directory") + ErrNilEntrypoint = errors.New("nil entrypoint") +) + +// Directory structure +type dirCache = map[string]*cachedEntrypoint + +type linkCache struct { + ep *Entrypoint // entrypoint of the link itself + target *cachedEntrypoint // target for the link +} + +// A single entry in directory cache, only one of entries below must be non-nil +type cachedEntrypoint struct { + // target data is stored and we've got valid entrypoint to it + stored *Entrypoint + + // Target is a link and contains modified data + link *linkCache + + // Target is a directory containing partially modified content + dir dirCache +} + +type cinodeFS struct { + c graphContext + maxLinkRedirects int + timeFunc func() time.Time + randSource io.Reader + + rootEP *cachedEntrypoint +} + +type CinodeFS = *cinodeFS + +func NewCinodeFS( + ctx context.Context, + be blenc.BE, + options ...CinodeFSOption, +) (*cinodeFS, error) { + if be == nil { + return nil, ErrInvalidBE + } + + ret := cinodeFS{ + maxLinkRedirects: DefaultMaxLinksRedirects, + timeFunc: time.Now, + randSource: rand.Reader, + c: graphContext{ + be: be, + writerInfos: map[string][]byte{}, + }, + } + + for _, opt := range options { + err := opt.apply(ctx, &ret) + if err != nil { + return nil, err + } + } + + return &ret, nil +} + +func (fs *cinodeFS) SetEntryFile( + ctx context.Context, + path []string, + data io.Reader, + mimeType string, +) (*Entrypoint, error) { + if mimeType == "" && len(path) > 0 { + // Detect mime type by file extension + mimeType = mime.TypeByExtension(filepath.Ext(path[len(path)-1])) + } + + ep, err := fs.CreateFileEntrypoint(ctx, data, mimeType) + if err != nil { + return nil, err + } + + err = fs.SetEntry(ctx, path, ep) + if err != nil { + return nil, err + } + + return ep, nil +} + +func (fs *cinodeFS) CreateFileEntrypoint( + ctx context.Context, + data io.Reader, + mimeType string, +) (*Entrypoint, error) { + var hw headWriter + + if mimeType == "" { + hw = newHeadWriter(512) + data = io.TeeReader(data, &hw) + } + + bn, key, _, err := fs.c.be.Create(ctx, blobtypes.Static, data) + if err != nil { + return nil, err + } + + if mimeType == "" { + mimeType = http.DetectContentType(hw.data) + } + + ep := entrypointFromBlobNameAndKey(bn, key) + ep.ep.MimeType = mimeType + return ep, nil +} + +func (fs *cinodeFS) SetEntry( + ctx context.Context, + path []string, + ep *Entrypoint, +) error { + rootEP, err := fs.setEntry(ctx, fs.rootEP, path, ep, 0) + if err != nil { + return err + } + fs.rootEP = rootEP + return nil +} + +func (fs *cinodeFS) setEntry( + ctx context.Context, + current *cachedEntrypoint, + path []string, + ep *Entrypoint, + linkDepth int, +) (*cachedEntrypoint, error) { + if current == nil { + // creating brand new path that does not exist yet + if len(path) == 0 { + return &cachedEntrypoint{stored: ep}, nil + } + // New empty directory + current = &cachedEntrypoint{dir: map[string]*cachedEntrypoint{}} + } + + // entry not yet loaded, we only know the entrypoint, load it then + loaded, err := fs.loadEntrypoint(ctx, current) + if err != nil { + return nil, err + } + current = loaded + + if current.link != nil { + if linkDepth >= fs.maxLinkRedirects { + return nil, ErrTooManyRedirects + } + + if _, hasWriterInfo := fs.c.writerInfos[current.link.ep.BlobName().String()]; !hasWriterInfo { + // We won't be able to update data behind given link + // TODO: This is false for recursive links, we only have to check this at the last level + return nil, ErrMissingWriterInfo + } + + // Update the target of the link + target, err := fs.setEntry(ctx, current.link.target, path, ep, linkDepth+1) + if err != nil { + return nil, err + } + current.link.target = target + return current, nil + } + + if len(path) == 0 { + // reached the final spot for the entrypoint, replace the current content + // TODO: This could be a very destructive change, should we do additional checks here? + // e.g. if there's a directory here, prevent replacing with a file + return &cachedEntrypoint{stored: ep}, nil + } + + if current.dir == nil { + // we need to have directory at this level + current = &cachedEntrypoint{dir: map[string]*cachedEntrypoint{}} + } + + if currentDirEntry, found := current.dir[path[0]]; found { + // Overwrite existing entry including descending into sub-dirs + updatedEntry, err := fs.setEntry(ctx, currentDirEntry, path[1:], ep, 0) + if err != nil { + return nil, err + } + current.dir[path[0]] = updatedEntry + return current, nil + } + + // No entry, create completely new path + newEntry, err := fs.setEntry(ctx, nil, path[1:], ep, 0) + if err != nil { + return nil, err + } + current.dir[path[0]] = newEntry + return current, nil +} + +func (fs *cinodeFS) loadEntrypoint( + ctx context.Context, + ep *cachedEntrypoint, +) ( + *cachedEntrypoint, + error, +) { + if ep.stored != nil { + // Data is behind some entrypoint, try to load it + if ep.stored.IsLink() { + return fs.loadEntrypointLink(ctx, ep.stored) + } + if ep.stored.IsDir() { + return fs.loadEntrypointDir(ctx, ep.stored) + } + } + + return ep, nil +} + +func (fs *cinodeFS) loadEntrypointLink( + ctx context.Context, + ep *Entrypoint, +) ( + *cachedEntrypoint, + error, +) { + msg := &protobuf.Entrypoint{} + err := fs.c.readProtobufMessage(ctx, ep, msg) + if err != nil { + return nil, err + } + + targetEP, err := entrypointFromProtobuf(msg) + if err != nil { + return nil, err + } + + return &cachedEntrypoint{ + link: &linkCache{ + ep: ep, + target: &cachedEntrypoint{ + stored: targetEP, + }, + }, + }, nil +} + +func (fs *cinodeFS) loadEntrypointDir( + ctx context.Context, + ep *Entrypoint, +) ( + *cachedEntrypoint, + error, +) { + msg := &protobuf.Directory{} + err := fs.c.readProtobufMessage(ctx, ep, msg) + if err != nil { + return nil, err + } + + dir := make(map[string]*cachedEntrypoint, len(msg.Entries)) + + for _, entry := range msg.Entries { + if entry.Name == "" { + return nil, errors.New("empty name") + } + if _, exists := dir[entry.Name]; exists { + return nil, errors.New("entry doubled") + } + + ep, err := entrypointFromProtobuf(entry.Ep) + if err != nil { + return nil, err + } + + dir[entry.Name] = &cachedEntrypoint{stored: ep} + } + + return &cachedEntrypoint{dir: dir}, nil +} + +func (fs *cinodeFS) Flush(ctx context.Context) error { + newRoot, err := fs.flush(ctx, fs.rootEP) + if err != nil { + return err + } + fs.rootEP = &cachedEntrypoint{stored: newRoot} + return nil +} + +func (fs *cinodeFS) flush(ctx context.Context, current *cachedEntrypoint) (*Entrypoint, error) { + if current.link != nil { + return fs.flushLink(ctx, current) + } + if current.dir != nil { + return fs.flushDir(ctx, current) + } + // already stored, no need to flush + return current.stored, nil +} + +func (fs *cinodeFS) flushLink(ctx context.Context, current *cachedEntrypoint) (*Entrypoint, error) { + target, err := fs.flush(ctx, current.link.target) + if err != nil { + return nil, err + } + + err = fs.c.updateProtobufMessage(ctx, current.link.ep, target.ep) + if err != nil { + return nil, err + } + + return current.link.ep, nil +} + +func (fs *cinodeFS) flushDir(ctx context.Context, current *cachedEntrypoint) (*Entrypoint, error) { + dir := protobuf.Directory{ + Entries: make([]*protobuf.Directory_Entry, 0, len(current.dir)), + } + + for name, entry := range current.dir { + flushed, err := fs.flush(ctx, entry) + if err != nil { + return nil, err + } + + dir.Entries = append(dir.Entries, &protobuf.Directory_Entry{ + Name: name, + Ep: flushed.ep, + }) + } + + sort.Slice(dir.Entries, func(i, j int) bool { + return dir.Entries[i].Name < dir.Entries[j].Name + }) + + ep, err := fs.c.createProtobufMessage(ctx, blobtypes.Static, &dir) + if err != nil { + return nil, err + } + ep.ep.MimeType = CinodeDirMimeType + + return ep, nil +} + +func (fs *cinodeFS) FindEntry(ctx context.Context, path []string) (*Entrypoint, error) { + return fs.findEntry(ctx, fs.rootEP, path, 0) +} + +func (fs *cinodeFS) findEntry( + ctx context.Context, + current *cachedEntrypoint, + path []string, + linkDepth int, +) (*Entrypoint, error) { + current, err := fs.loadEntrypoint(ctx, current) + if err != nil { + return nil, err + } + + if current.link != nil { + if linkDepth >= fs.maxLinkRedirects { + return nil, ErrTooManyRedirects + } + return fs.findEntry(ctx, current.link.target, path, linkDepth+1) + } + + if current.dir != nil { + return fs.findEntryInDir(ctx, current.dir, path) + } + + if len(path) > 0 { + return nil, ErrNotADirectory + } + + return current.stored, nil +} + +func (fs *cinodeFS) findEntryInDir(ctx context.Context, dir dirCache, path []string) (*Entrypoint, error) { + if len(path) == 0 { + return nil, ErrModifiedDirectory + } + + entry, found := dir[path[0]] + if !found { + return nil, ErrEntryNotFound + } + + return fs.findEntry(ctx, entry, path[1:], 0) +} + +func (fs *cinodeFS) DeleteEntry(ctx context.Context, path []string) error { + // Entry removal is done on the parent level, we find the parent directory + // and remove the entry from its list + + if len(path) == 0 { + return ErrCantDeleteRoot + } + + newRoot, err := fs.deleteEntry( + ctx, + fs.rootEP, + path[:len(path)-1], + path[len(path)-1], + 0, + ) + if err != nil { + return err + } + fs.rootEP = newRoot + return nil +} + +func (fs *cinodeFS) deleteEntry( + ctx context.Context, + current *cachedEntrypoint, + path []string, + entryName string, + linkDepth int, +) ( + *cachedEntrypoint, + error, +) { + current, err := fs.loadEntrypoint(ctx, current) + if err != nil { + return nil, err + } + + if current.link != nil { + if linkDepth >= fs.maxLinkRedirects { + return nil, ErrTooManyRedirects + } + + if _, hasWriterInfo := fs.c.writerInfos[current.link.ep.BlobName().String()]; !hasWriterInfo { + // We won't be able to update data behind given link + // TODO: This is false for recursive links, we only have to check this at the last level + return nil, ErrMissingWriterInfo + } + + newTarget, err := fs.deleteEntry( + ctx, + current.link.target, + path, + entryName, + linkDepth+1, + ) + if err != nil { + return nil, err + } + current.link.target = newTarget + return current, nil + } + + if current.dir == nil { + return nil, ErrNotADirectory + } + + if len(path) == 0 { + // Got to the target directory, try to remove the entry + if _, found := current.dir[entryName]; !found { + return nil, ErrEntryNotFound + } + delete(current.dir, entryName) + return current, nil + } + + // Not yet at the target, descend to sub-directory + subDir, found := current.dir[path[0]] + if !found { + return nil, ErrEntryNotFound + } + + subDir, err = fs.deleteEntry(ctx, subDir, path[1:], entryName, 0) + if err != nil { + return nil, err + } + + current.dir[path[0]] = subDir + return current, nil +} + +func (fs *cinodeFS) generateNewDynamicLinkEntrypoint() (*Entrypoint, error) { + // Generate new entrypoint link data but do not yet store it in datastore + link, err := dynamiclink.Create(fs.randSource) + if err != nil { + return nil, err + } + + bn := link.BlobName() + key := link.EncryptionKey() + + fs.c.writerInfos[bn.String()] = link.AuthInfo() + + return entrypointFromBlobNameAndKey(bn, key), nil +} + +func (fs *cinodeFS) OpenEntrypointData(ctx context.Context, ep *Entrypoint) (io.ReadCloser, error) { + if ep == nil { + return nil, ErrNilEntrypoint + } + + return fs.c.getDataReader(ctx, ep) +} diff --git a/pkg/structure/graph/cinodefs_blackbox_test.go b/pkg/structure/graph/cinodefs_blackbox_test.go new file mode 100644 index 0000000..aeb568c --- /dev/null +++ b/pkg/structure/graph/cinodefs_blackbox_test.go @@ -0,0 +1,111 @@ +package graph_test + +import ( + "context" + "fmt" + "io" + "strings" + "testing" + + "github.com/cinode/go/pkg/blenc" + "github.com/cinode/go/pkg/datastore" + "github.com/cinode/go/pkg/structure/graph" + "github.com/stretchr/testify/require" +) + +func TestSampleScenario(t *testing.T) { + ds := datastore.InMemory() + be := blenc.FromDatastore(ds) + + ctx := context.Background() + + fs, err := graph.NewCinodeFS( + ctx, + be, + graph.NewRootDynamicLink(), + ) + require.NoError(t, err) + require.NotNil(t, fs) + + path1 := []string{"dir", "subdir", "file.txt"} + + ep1, err := fs.SetEntryFile(ctx, + path1, + strings.NewReader("Hello world!"), + "", + ) + require.NoError(t, err) + require.NotNil(t, ep1) + + ep2, err := fs.FindEntry( + ctx, + path1, + ) + require.NoError(t, err) + require.NotNil(t, ep2) + + require.Equal(t, ep1.String(), ep2.String()) + + // Get the entrypoint to the root + ep3, err := fs.FindEntry(ctx, []string{}) + require.ErrorIs(t, err, graph.ErrModifiedDirectory) + require.Nil(t, ep3) + + err = fs.Flush(ctx) + require.NoError(t, err) + + contentMap := make([]struct { + path []string + content string + }, 1000) + + for i := 0; i < 1000; i++ { + contentMap[i].path = []string{ + fmt.Sprintf("dir%d", i%7), + fmt.Sprintf("subdir%d", i%19), + fmt.Sprintf("file%d.txt", i), + } + contentMap[i].content = fmt.Sprintf("Hello world! from file %d!", i) + } + + for _, c := range contentMap { + _, err := fs.SetEntryFile(ctx, + c.path, + strings.NewReader(c.content), + "", + ) + require.NoError(t, err) + } + + checkContentMap := func(fs graph.CinodeFS) { + for _, c := range contentMap { + ep, err := fs.FindEntry(ctx, c.path) + require.NoError(t, err) + require.Contains(t, ep.MimeType(), "text/plain") + + rc, err := fs.OpenEntrypointData(ctx, ep) + require.NoError(t, err) + defer rc.Close() + + data, err := io.ReadAll(rc) + require.NoError(t, err) + + require.Equal(t, c.content, string(data)) + } + } + checkContentMap(fs) + + err = fs.Flush(ctx) + require.NoError(t, err) + + checkContentMap(fs) + + // fs2, err := graph.NewCinodeFS( + // ctx, + // blenc.FromDatastore(ds), + // graph.RootEntrypointString() + // ) + + // TODO: + // * reopening the datastore +} diff --git a/pkg/structure/graph/cinodefs_options.go b/pkg/structure/graph/cinodefs_options.go new file mode 100644 index 0000000..a86847b --- /dev/null +++ b/pkg/structure/graph/cinodefs_options.go @@ -0,0 +1,84 @@ +package graph + +import ( + "context" + "io" + "time" +) + +const ( + DefaultMaxLinksRedirects = 10 +) + +type CinodeFSOption interface { + apply(ctx context.Context, fs *cinodeFS) error +} + +type optionFunc func(ctx context.Context, fs *cinodeFS) error + +func (f optionFunc) apply(ctx context.Context, fs *cinodeFS) error { + return f(ctx, fs) +} + +func MaxLinkRedirects(maxLinkRedirects int) CinodeFSOption { + return optionFunc(func(ctx context.Context, fs *cinodeFS) error { + fs.maxLinkRedirects = maxLinkRedirects + return nil + }) +} + +func RootEntrypoint(ep *Entrypoint) CinodeFSOption { + return optionFunc(func(ctx context.Context, fs *cinodeFS) error { + fs.rootEP = &cachedEntrypoint{stored: ep} + return nil + }) +} + +func RootEntrypointString(eps string) CinodeFSOption { + ep, err := EntrypointFromString(eps) + if err != nil { + return optionFunc(func(ctx context.Context, fs *cinodeFS) error { + return err + }) + } + return RootEntrypoint(ep) +} + +func TimeFunc(f func() time.Time) CinodeFSOption { + return optionFunc(func(ctx context.Context, fs *cinodeFS) error { + fs.timeFunc = f + return nil + }) +} + +func RandSource(r io.Reader) CinodeFSOption { + return optionFunc(func(ctx context.Context, fs *cinodeFS) error { + fs.randSource = r + return nil + }) +} + +// NewRootDynamicLink option can be used to create completely new, random +// dynamic link as the root +func NewRootDynamicLink() CinodeFSOption { + return optionFunc(func(ctx context.Context, fs *cinodeFS) error { + newLinkEntrypoint, err := fs.generateNewDynamicLinkEntrypoint() + if err != nil { + return err + } + + // Generate a simple dummy structure consisting of a root link + // and an empty directory, all the entries are in-memory upon + // creation and have to be flushed first to generate any + // blobs + fs.rootEP = &cachedEntrypoint{ + link: &linkCache{ + ep: newLinkEntrypoint, + target: &cachedEntrypoint{ + dir: map[string]*cachedEntrypoint{}, + }, + }, + } + return nil + }) +} diff --git a/pkg/structure/graph/context.go b/pkg/structure/graph/context.go new file mode 100644 index 0000000..fc46b79 --- /dev/null +++ b/pkg/structure/graph/context.go @@ -0,0 +1,140 @@ +package graph + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + + "github.com/cinode/go/pkg/blenc" + "github.com/cinode/go/pkg/common" + "google.golang.org/protobuf/proto" +) + +var ( + ErrMissingKeyInfo = errors.New("missing key info") + ErrMissingWriterInfo = errors.New("missing writer info") +) + +type graphContext struct { + // blenc layer used in the graph + be blenc.BE + + // known writer info data + writerInfos map[string][]byte +} + +// Get symmetric encryption key for given entrypoint. +// +// Note: Currently the key will be stored inside entrypoint data, +// but more advanced methods of obtaining the key may be added +// through this function in the future. +func (c *graphContext) keyFromEntrypoint( + ctx context.Context, + ep *Entrypoint, +) (common.BlobKey, error) { + if ep.ep == nil || + ep.ep.KeyInfo == nil || + ep.ep.KeyInfo.Key == nil { + return common.BlobKey{}, ErrMissingKeyInfo + } + return common.BlobKeyFromBytes(ep.ep.GetKeyInfo().GetKey()), nil +} + +// open io.ReadCloser for data behind given entrypoint +func (c *graphContext) getDataReader( + ctx context.Context, + ep *Entrypoint, +) ( + io.ReadCloser, + error, +) { + key, err := c.keyFromEntrypoint(ctx, ep) + if err != nil { + return nil, err + } + rc, err := c.be.Open(ctx, ep.BlobName(), key) + if err != nil { + return nil, fmt.Errorf("failed to open blob: %w", err) + } + return rc, nil +} + +// return data behind entrypoint +func (c *graphContext) readProtobufMessage( + ctx context.Context, + ep *Entrypoint, + msg proto.Message, +) error { + rc, err := c.getDataReader(ctx, ep) + if err != nil { + return err + } + defer rc.Close() + + data, err := io.ReadAll(rc) + if err != nil { + return fmt.Errorf("failed to read blob: %w", err) + } + + err = proto.Unmarshal(data, msg) + if err != nil { + return fmt.Errorf("malformed data: %w", err) + } + + return nil +} + +func (c *graphContext) createProtobufMessage( + ctx context.Context, + blobType common.BlobType, + msg proto.Message, +) ( + *Entrypoint, + error, +) { + data, err := proto.Marshal(msg) + if err != nil { + return nil, fmt.Errorf("serialization failed: %w", err) + } + + bn, key, wi, err := c.be.Create(ctx, blobType, bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("write failed: %w", err) + } + + if wi != nil { + c.writerInfos[bn.String()] = wi + } + + return entrypointFromBlobNameAndKey(bn, key), nil +} + +func (c *graphContext) updateProtobufMessage( + ctx context.Context, + ep *Entrypoint, + msg proto.Message, +) error { + wi, found := c.writerInfos[ep.BlobName().String()] + if !found { + return ErrMissingWriterInfo + } + + key, err := c.keyFromEntrypoint(ctx, ep) + if err != nil { + return err + } + + data, err := proto.Marshal(msg) + if err != nil { + return fmt.Errorf("serialization failed: %w", err) + } + + err = c.be.Update(ctx, ep.BlobName(), wi, key, bytes.NewReader(data)) + if err != nil { + return fmt.Errorf("write failed: %w", err) + } + + return nil +} diff --git a/pkg/structure/graph/dir.go b/pkg/structure/graph/dir.go new file mode 100644 index 0000000..ab66810 --- /dev/null +++ b/pkg/structure/graph/dir.go @@ -0,0 +1,157 @@ +package graph + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + "github.com/cinode/go/pkg/blobtypes" + "github.com/cinode/go/pkg/structure/internal/protobuf" +) + +const ( + CinodeDirMimeType = "application/cinode-dir" +) + +var ( + // Returned when an entry does not exist in a directory + ErrEntryNotFound = errors.New("entry not found") + + // Returned when there's a directory read error + ErrCantReadDirectory = errors.New("can not read directory") + + // Returned when directory blob was read correctly but the data is corrupted + ErrInvalidDirectoryData = errors.New("invalid directory data") + + ErrCantWriteDirectory = errors.New("can not write directory") +) + +type Dir struct { + entries map[string]*Entrypoint +} + +func NewEmptyDir() *Dir { + return &Dir{ + entries: map[string]*Entrypoint{}, + } +} + +func LoadDir(ctx context.Context, ep *Entrypoint, c *graphContext) (*Dir, error) { + dirMessage := protobuf.Directory{} + + err := c.readProtobufMessage(ctx, ep, &dirMessage) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidDirectoryData, err) + } + + dir := &Dir{ + entries: make(map[string]*Entrypoint, len(dirMessage.Entries)), + } + + for _, entry := range dirMessage.Entries { + if entry.Name == "" { + return nil, fmt.Errorf("%w: entry with empty name", ErrInvalidDirectoryData) + } + ep, err := entrypointFromProtobuf(entry.Ep) + if err != nil { + return nil, fmt.Errorf( + "%w: invalid entrypoint for %s entry: %w", + ErrInvalidDirectoryData, entry.Name, err, + ) + } + if _, found := dir.entries[entry.Name]; found { + return nil, fmt.Errorf( + "%w: invalid entry for %s: duplicate found", + ErrInvalidDirectoryData, + entry.Name, + ) + } + dir.entries[entry.Name] = ep + } + + return dir, nil +} + +func (d *Dir) FindEntry(n string) (*Entrypoint, error) { + e, found := d.entries[n] + if !found { + return nil, ErrEntryNotFound + } + return e, nil +} + +type DirEntriesFilter struct { + NamePrefix string + NameSuffix string +} + +func (f *DirEntriesFilter) matches(name string, ep *Entrypoint) bool { + if f == nil { + return true + } + + if !strings.HasPrefix(name, f.NamePrefix) { + return false + } + if !strings.HasSuffix(name, f.NameSuffix) { + return false + } + + return true +} + +type DirEntriesFunc func(name string, ep *Entrypoint) + +func (d *Dir) EnumerateEntries( + ctx context.Context, + filter *DirEntriesFilter, + callback DirEntriesFunc, +) error { + for name, entry := range d.entries { + if ctx.Err() != nil { + return ctx.Err() + } + if filter.matches(name, entry) { + callback(name, entry) + } + } + return nil +} + +func (d *Dir) SetEntry(n string, ep *Entrypoint) { + d.entries[n] = ep +} + +func (d *Dir) DeleteEntry(n string) error { + if _, found := d.entries[n]; !found { + return ErrEntryNotFound + } + delete(d.entries, n) + return nil +} + +func (d *Dir) Store(ctx context.Context, c *graphContext) (*Entrypoint, error) { + dir := protobuf.Directory{ + Entries: make([]*protobuf.Directory_Entry, len(d.entries)), + } + + for name, entry := range d.entries { + dir.Entries = append(dir.Entries, &protobuf.Directory_Entry{ + Name: name, + Ep: entry.ep, + }) + } + + sort.Slice(dir.Entries, func(i, j int) bool { + return dir.Entries[i].Name < dir.Entries[j].Name + }) + + ep, err := c.createProtobufMessage(ctx, blobtypes.Static, &dir) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrCantWriteDirectory, err) + } + + return ep, nil +} diff --git a/pkg/structure/graph/entrypoint.go b/pkg/structure/graph/entrypoint.go new file mode 100644 index 0000000..02c96fc --- /dev/null +++ b/pkg/structure/graph/entrypoint.go @@ -0,0 +1,126 @@ +package graph + +import ( + "errors" + "fmt" + "time" + + "github.com/cinode/go/pkg/blobtypes" + "github.com/cinode/go/pkg/common" + "github.com/cinode/go/pkg/structure/internal/protobuf" + "github.com/cinode/go/pkg/utilities/golang" + "github.com/jbenet/go-base58" + "google.golang.org/protobuf/proto" +) + +var ( + ErrInvalidEntrypointData = errors.New("invalid entrypoint data") + ErrInvalidEntrypointDataParse = fmt.Errorf("%w: protobuf parse error", ErrInvalidEntrypointData) + ErrInvalidEntrypointDataLinkMimetype = fmt.Errorf("%w: link can not have mimetype set", ErrInvalidEntrypointData) + ErrInvalidEntrypointDataNil = fmt.Errorf("%w: nil data", ErrInvalidEntrypointData) + ErrInvalidEntrypointTime = errors.New("time validation failed") + ErrExpired = fmt.Errorf("%w: entry expired", ErrInvalidEntrypointTime) + ErrNotYetValid = fmt.Errorf("%w: entry not yet valid", ErrInvalidEntrypointTime) +) + +type Entrypoint struct { + ep *protobuf.Entrypoint + bn common.BlobName +} + +func EntrypointFromString(s string) (*Entrypoint, error) { + if len(s) == 0 { + return nil, fmt.Errorf("%w: empty string", ErrInvalidEntrypointData) + } + + b := base58.Decode(s) + if len(b) == 0 { + return nil, fmt.Errorf("%w: not a base58 string", ErrInvalidEntrypointData) + } + + return EntrypointFromBytes(b) +} + +func EntrypointFromBytes(b []byte) (*Entrypoint, error) { + data := protobuf.Entrypoint{} + + err := proto.Unmarshal(b, &data) + if err != nil { + return nil, fmt.Errorf("%w: %s", ErrInvalidEntrypointDataParse, err) + } + + return entrypointFromProtobuf(&data) +} + +func entrypointFromProtobuf(data *protobuf.Entrypoint) (*Entrypoint, error) { + if data == nil { + return nil, ErrInvalidEntrypointDataNil + } + + ret := Entrypoint{ep: data} + + // Extract blob name from entrypoint + bn, err := common.BlobNameFromBytes(data.BlobName) + if err != nil { + return nil, fmt.Errorf("%w: %s", ErrInvalidEntrypointData, err) + } + ret.bn = bn + + // Links must not have mimetype set + if ret.IsLink() && data.MimeType != "" { + return nil, ErrInvalidEntrypointDataLinkMimetype + } + + return &ret, nil +} + +func entrypointFromBlobNameAndKey(bn common.BlobName, key common.BlobKey) *Entrypoint { + return &Entrypoint{ + ep: &protobuf.Entrypoint{ + BlobName: bn.Bytes(), + KeyInfo: &protobuf.KeyInfo{ + Key: key.Bytes(), + }, + }, + bn: bn, + } +} + +func (e *Entrypoint) String() string { + return base58.Encode(e.Bytes()) +} + +func (e *Entrypoint) Bytes() []byte { + return golang.Must(proto.Marshal(e.ep)) +} + +func (e *Entrypoint) BlobName() common.BlobName { + return e.bn +} + +func (e *Entrypoint) IsLink() bool { + return e.bn.Type() == blobtypes.DynamicLink +} + +func (e *Entrypoint) IsDir() bool { + return e.ep.MimeType == CinodeDirMimeType +} + +func (e *Entrypoint) MimeType() string { + return e.ep.MimeType +} + +func (e *Entrypoint) IsValid(now time.Time) error { + nowMicro := now.UnixMicro() + if e.ep.NotValidBeforeUnixMicro != 0 { + if e.ep.NotValidBeforeUnixMicro > nowMicro { + return ErrNotYetValid + } + } + if e.ep.NotValidAfterUnixMicro != 0 { + if e.ep.NotValidAfterUnixMicro < nowMicro { + return ErrExpired + } + } + return nil +} diff --git a/pkg/structure/graph/headwriter.go b/pkg/structure/graph/headwriter.go new file mode 100644 index 0000000..0e854b9 --- /dev/null +++ b/pkg/structure/graph/headwriter.go @@ -0,0 +1,27 @@ +package graph + +type headWriter struct { + limit int + data []byte +} + +func newHeadWriter(limit int) headWriter { + return headWriter{ + limit: limit, + data: make([]byte, limit), + } +} + +func (h *headWriter) Write(b []byte) (int, error) { + if len(h.data) >= h.limit { + return len(b), nil + } + + if len(h.data)+len(b) > h.limit { + h.data = append(h.data, b[:h.limit-len(h.data)]...) + return len(b), nil + } + + h.data = append(h.data, b...) + return len(b), nil +} diff --git a/pkg/structure/graph/link.go b/pkg/structure/graph/link.go new file mode 100644 index 0000000..ff006fb --- /dev/null +++ b/pkg/structure/graph/link.go @@ -0,0 +1,66 @@ +package graph + +import ( + "context" + "fmt" + + "github.com/cinode/go/pkg/blobtypes" + "github.com/cinode/go/pkg/structure/internal/protobuf" +) + +type Link struct { + ep *Entrypoint + tep *Entrypoint +} + +func NewLink( + ctx context.Context, + targetEP *Entrypoint, + c *graphContext, +) (*Link, error) { + ep, err := c.createProtobufMessage(ctx, blobtypes.DynamicLink, targetEP.ep) + if err != nil { + return nil, err + } + + return &Link{ + ep: ep, + tep: targetEP, + }, nil +} + +func OpenLink( + ctx context.Context, + ep *Entrypoint, + c *graphContext, +) (*Link, error) { + tepRaw := protobuf.Entrypoint{} + + err := c.readProtobufMessage(ctx, ep, &tepRaw) + if err != nil { + return nil, err + } + + tep, err := entrypointFromProtobuf(&tepRaw) + if err != nil { + return nil, err + } + + return &Link{ + ep: ep, + tep: tep, + }, nil +} + +func (l *Link) Update(ctx context.Context, tep *Entrypoint, c *graphContext) error { + err := c.updateProtobufMessage( + ctx, + l.ep, + tep.ep, + ) + if err != nil { + return fmt.Errorf("link update failed: %w", err) + } + l.tep = tep + return nil +} diff --git a/pkg/structure/graph/writerinfo.go b/pkg/structure/graph/writerinfo.go new file mode 100644 index 0000000..69c5c0a --- /dev/null +++ b/pkg/structure/graph/writerinfo.go @@ -0,0 +1,47 @@ +package graph + +import ( + "errors" + "fmt" + + "github.com/cinode/go/pkg/structure/internal/protobuf" + "github.com/jbenet/go-base58" + "google.golang.org/protobuf/proto" +) + +var ( + ErrInvalidWriterInfoData = errors.New("invalid writer info data") + ErrInvalidWriterInfoDataParse = fmt.Errorf("%w: protobuf parse error", ErrInvalidWriterInfoData) +) + +type WriterInfo struct { + wi *protobuf.WriterInfo +} + +func WriterInfoFromString(s string) (WriterInfo, error) { + if len(s) == 0 { + return WriterInfo{}, fmt.Errorf("%w: empty string", ErrInvalidWriterInfoData) + } + + b := base58.Decode(s) + if len(b) == 0 { + return WriterInfo{}, fmt.Errorf("%w: not a base58 string", ErrInvalidWriterInfoData) + } + + return WriterInfoFromBytes(b) +} + +func WriterInfoFromBytes(b []byte) (WriterInfo, error) { + data := protobuf.WriterInfo{} + + err := proto.Unmarshal(b, &data) + if err != nil { + return WriterInfo{}, fmt.Errorf("%w: %s", ErrInvalidWriterInfoDataParse, err) + } + + return writerInfoFromProtobuf(&data) +} + +func writerInfoFromProtobuf(data *protobuf.WriterInfo) (WriterInfo, error) { + return WriterInfo{wi: data}, nil +} diff --git a/pkg/utilities/golang/assert.go b/pkg/utilities/golang/assert.go new file mode 100644 index 0000000..128600d --- /dev/null +++ b/pkg/utilities/golang/assert.go @@ -0,0 +1,7 @@ +package golang + +func Assert(b bool, message string) { + if !b { + panic("Assertion failed: " + message) + } +} From 187454eb898c5528eb221ffae2ab10d6cc968fcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Mon, 9 Oct 2023 00:00:42 +0200 Subject: [PATCH 07/29] Refactor cinodeFs traversal code * Introduce abstract node structure, all recursive functions are now realized through those nodes and implementations of node interface method implementations. * Introduce generic traversal logic - a common code paths are now used for graph traversal and only the final step that deals with the target node is customized for specific logic. That way bug fixing and testing can be done on a common path improving overall code quality. --- .vscode/settings.json | 1 + pkg/structure/graph/cinodefs.go | 520 +++++------------- pkg/structure/graph/cinodefs_blackbox_test.go | 361 +++++++++--- pkg/structure/graph/cinodefs_options.go | 66 ++- pkg/structure/graph/cinodefs_traverse.go | 70 +++ pkg/structure/graph/context.go | 27 +- pkg/structure/graph/dir.go | 157 ------ pkg/structure/graph/entrypoint.go | 30 +- pkg/structure/graph/entrypoint_options.go | 67 +++ pkg/structure/graph/headwriter.go | 16 + pkg/structure/graph/link.go | 66 --- pkg/structure/graph/node.go | 81 +++ pkg/structure/graph/node_directory.go | 229 ++++++++ pkg/structure/graph/node_file.go | 58 ++ pkg/structure/graph/node_link.go | 121 ++++ pkg/structure/graph/node_unloaded.go | 135 +++++ pkg/structure/graph/writerinfo.go | 27 + 17 files changed, 1356 insertions(+), 676 deletions(-) create mode 100644 pkg/structure/graph/cinodefs_traverse.go delete mode 100644 pkg/structure/graph/dir.go create mode 100644 pkg/structure/graph/entrypoint_options.go delete mode 100644 pkg/structure/graph/link.go create mode 100644 pkg/structure/graph/node.go create mode 100644 pkg/structure/graph/node_directory.go create mode 100644 pkg/structure/graph/node_file.go create mode 100644 pkg/structure/graph/node_link.go create mode 100644 pkg/structure/graph/node_unloaded.go diff --git a/.vscode/settings.json b/.vscode/settings.json index 3f5bd96..df3a0f0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -24,6 +24,7 @@ "securefifo", "shogo", "stretchr", + "subdir", "testblobs", "testvectors", "validatingreader" diff --git a/pkg/structure/graph/cinodefs.go b/pkg/structure/graph/cinodefs.go index 1eb4c55..f505a46 100644 --- a/pkg/structure/graph/cinodefs.go +++ b/pkg/structure/graph/cinodefs.go @@ -1,3 +1,19 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package graph import ( @@ -8,7 +24,6 @@ import ( "mime" "net/http" "path/filepath" - "sort" "time" "github.com/cinode/go/pkg/blenc" @@ -18,35 +33,26 @@ import ( ) var ( - ErrInvalidBE = errors.New("invalid BE argument") - ErrCantOpenDir = errors.New("can not open directory") - ErrTooManyRedirects = errors.New("too many link redirects") - ErrCantComputeBlobKey = errors.New("can not compute blob keys") - ErrModifiedDirectory = errors.New("can not get entrypoint for a directory, unsaved content") - ErrCantDeleteRoot = errors.New("can not delete root object") - ErrNotADirectory = errors.New("entry is not a directory") - ErrNilEntrypoint = errors.New("nil entrypoint") + ErrInvalidBE = errors.New("invalid BE argument") + ErrCantOpenDir = errors.New("can not open directory") + ErrTooManyRedirects = errors.New("too many link redirects") + ErrCantComputeBlobKey = errors.New("can not compute blob keys") + ErrModifiedDirectory = errors.New("can not get entrypoint for a directory, unsaved content") + ErrCantDeleteRoot = errors.New("can not delete root object") + ErrNotADirectory = errors.New("entry is not a directory") + ErrNotALink = errors.New("entry is not a link") + ErrNilEntrypoint = errors.New("nil entrypoint") + ErrEmptyName = errors.New("entry name can not be empty") + ErrDuplicateEntry = errors.New("duplicate entry") + ErrEntryNotFound = errors.New("entry not found") + ErrCantReadDirectory = errors.New("can not read directory") + ErrInvalidDirectoryData = errors.New("invalid directory data") + ErrCantWriteDirectory = errors.New("can not write directory") ) -// Directory structure -type dirCache = map[string]*cachedEntrypoint - -type linkCache struct { - ep *Entrypoint // entrypoint of the link itself - target *cachedEntrypoint // target for the link -} - -// A single entry in directory cache, only one of entries below must be non-nil -type cachedEntrypoint struct { - // target data is stored and we've got valid entrypoint to it - stored *Entrypoint - - // Target is a link and contains modified data - link *linkCache - - // Target is a directory containing partially modified content - dir dirCache -} +const ( + CinodeDirMimeType = "application/cinode-dir" +) type cinodeFS struct { c graphContext @@ -54,7 +60,7 @@ type cinodeFS struct { timeFunc func() time.Time randSource io.Reader - rootEP *cachedEntrypoint + rootEP node } type CinodeFS = *cinodeFS @@ -92,14 +98,18 @@ func (fs *cinodeFS) SetEntryFile( ctx context.Context, path []string, data io.Reader, - mimeType string, + opts ...EntrypointOption, ) (*Entrypoint, error) { - if mimeType == "" && len(path) > 0 { - // Detect mime type by file extension - mimeType = mime.TypeByExtension(filepath.Ext(path[len(path)-1])) + protoEntrypoint, err := protoEntrypointFromOptions(ctx, opts...) + if err != nil { + return nil, err + } + if protoEntrypoint.MimeType == "" && len(path) > 0 { + // Try detecting mime type from filename extension + protoEntrypoint.MimeType = mime.TypeByExtension(filepath.Ext(path[len(path)-1])) } - ep, err := fs.CreateFileEntrypoint(ctx, data, mimeType) + ep, err := fs.createFileEntrypoint(ctx, data, protoEntrypoint) if err != nil { return nil, err } @@ -115,11 +125,25 @@ func (fs *cinodeFS) SetEntryFile( func (fs *cinodeFS) CreateFileEntrypoint( ctx context.Context, data io.Reader, - mimeType string, + opts ...EntrypointOption, +) (*Entrypoint, error) { + ep, err := protoEntrypointFromOptions(ctx, opts...) + if err != nil { + return nil, err + } + + return fs.createFileEntrypoint(ctx, data, ep) +} + +func (fs *cinodeFS) createFileEntrypoint( + ctx context.Context, + data io.Reader, + protoEntrypoint *protobuf.Entrypoint, ) (*Entrypoint, error) { var hw headWriter - if mimeType == "" { + if protoEntrypoint.MimeType == "" { + // detect mimetype from the content hw = newHeadWriter(512) data = io.TeeReader(data, &hw) } @@ -129,12 +153,11 @@ func (fs *cinodeFS) CreateFileEntrypoint( return nil, err } - if mimeType == "" { - mimeType = http.DetectContentType(hw.data) + if protoEntrypoint.MimeType == "" { + protoEntrypoint.MimeType = http.DetectContentType(hw.data) } - ep := entrypointFromBlobNameAndKey(bn, key) - ep.ep.MimeType = mimeType + ep := entrypointFromBlobNameKeyAndProtoEntrypoint(bn, key, protoEntrypoint) return ep, nil } @@ -143,389 +166,142 @@ func (fs *cinodeFS) SetEntry( path []string, ep *Entrypoint, ) error { - rootEP, err := fs.setEntry(ctx, fs.rootEP, path, ep, 0) - if err != nil { - return err - } - fs.rootEP = rootEP - return nil -} - -func (fs *cinodeFS) setEntry( - ctx context.Context, - current *cachedEntrypoint, - path []string, - ep *Entrypoint, - linkDepth int, -) (*cachedEntrypoint, error) { - if current == nil { - // creating brand new path that does not exist yet - if len(path) == 0 { - return &cachedEntrypoint{stored: ep}, nil - } - // New empty directory - current = &cachedEntrypoint{dir: map[string]*cachedEntrypoint{}} - } - - // entry not yet loaded, we only know the entrypoint, load it then - loaded, err := fs.loadEntrypoint(ctx, current) - if err != nil { - return nil, err - } - current = loaded - - if current.link != nil { - if linkDepth >= fs.maxLinkRedirects { - return nil, ErrTooManyRedirects + whenReached := func( + ctx context.Context, + current node, + isWriteable bool, + ) (node, dirtyState, error) { + if !isWriteable { + return nil, 0, ErrMissingWriterInfo } - - if _, hasWriterInfo := fs.c.writerInfos[current.link.ep.BlobName().String()]; !hasWriterInfo { - // We won't be able to update data behind given link - // TODO: This is false for recursive links, we only have to check this at the last level - return nil, ErrMissingWriterInfo - } - - // Update the target of the link - target, err := fs.setEntry(ctx, current.link.target, path, ep, linkDepth+1) - if err != nil { - return nil, err - } - current.link.target = target - return current, nil + return &nodeUnloaded{ep: *ep}, dsDirty, nil } - if len(path) == 0 { - // reached the final spot for the entrypoint, replace the current content - // TODO: This could be a very destructive change, should we do additional checks here? - // e.g. if there's a directory here, prevent replacing with a file - return &cachedEntrypoint{stored: ep}, nil - } - - if current.dir == nil { - // we need to have directory at this level - current = &cachedEntrypoint{dir: map[string]*cachedEntrypoint{}} - } - - if currentDirEntry, found := current.dir[path[0]]; found { - // Overwrite existing entry including descending into sub-dirs - updatedEntry, err := fs.setEntry(ctx, currentDirEntry, path[1:], ep, 0) - if err != nil { - return nil, err - } - current.dir[path[0]] = updatedEntry - return current, nil - } - - // No entry, create completely new path - newEntry, err := fs.setEntry(ctx, nil, path[1:], ep, 0) - if err != nil { - return nil, err - } - current.dir[path[0]] = newEntry - return current, nil -} - -func (fs *cinodeFS) loadEntrypoint( - ctx context.Context, - ep *cachedEntrypoint, -) ( - *cachedEntrypoint, - error, -) { - if ep.stored != nil { - // Data is behind some entrypoint, try to load it - if ep.stored.IsLink() { - return fs.loadEntrypointLink(ctx, ep.stored) - } - if ep.stored.IsDir() { - return fs.loadEntrypointDir(ctx, ep.stored) - } - } - - return ep, nil -} - -func (fs *cinodeFS) loadEntrypointLink( - ctx context.Context, - ep *Entrypoint, -) ( - *cachedEntrypoint, - error, -) { - msg := &protobuf.Entrypoint{} - err := fs.c.readProtobufMessage(ctx, ep, msg) - if err != nil { - return nil, err - } - - targetEP, err := entrypointFromProtobuf(msg) - if err != nil { - return nil, err - } - - return &cachedEntrypoint{ - link: &linkCache{ - ep: ep, - target: &cachedEntrypoint{ - stored: targetEP, - }, + return fs.traverseGraph( + ctx, + path, + traverseOptions{ + createNodes: true, + maxLinkDepth: fs.maxLinkRedirects, }, - }, nil -} - -func (fs *cinodeFS) loadEntrypointDir( - ctx context.Context, - ep *Entrypoint, -) ( - *cachedEntrypoint, - error, -) { - msg := &protobuf.Directory{} - err := fs.c.readProtobufMessage(ctx, ep, msg) - if err != nil { - return nil, err - } - - dir := make(map[string]*cachedEntrypoint, len(msg.Entries)) - - for _, entry := range msg.Entries { - if entry.Name == "" { - return nil, errors.New("empty name") - } - if _, exists := dir[entry.Name]; exists { - return nil, errors.New("entry doubled") - } - - ep, err := entrypointFromProtobuf(entry.Ep) - if err != nil { - return nil, err - } - - dir[entry.Name] = &cachedEntrypoint{stored: ep} - } - - return &cachedEntrypoint{dir: dir}, nil + whenReached, + ) } func (fs *cinodeFS) Flush(ctx context.Context) error { - newRoot, err := fs.flush(ctx, fs.rootEP) + newRoot, err := fs.rootEP.flush(ctx, &fs.c) if err != nil { return err } - fs.rootEP = &cachedEntrypoint{stored: newRoot} - return nil -} -func (fs *cinodeFS) flush(ctx context.Context, current *cachedEntrypoint) (*Entrypoint, error) { - if current.link != nil { - return fs.flushLink(ctx, current) - } - if current.dir != nil { - return fs.flushDir(ctx, current) - } - // already stored, no need to flush - return current.stored, nil -} - -func (fs *cinodeFS) flushLink(ctx context.Context, current *cachedEntrypoint) (*Entrypoint, error) { - target, err := fs.flush(ctx, current.link.target) - if err != nil { - return nil, err - } - - err = fs.c.updateProtobufMessage(ctx, current.link.ep, target.ep) - if err != nil { - return nil, err - } - - return current.link.ep, nil -} - -func (fs *cinodeFS) flushDir(ctx context.Context, current *cachedEntrypoint) (*Entrypoint, error) { - dir := protobuf.Directory{ - Entries: make([]*protobuf.Directory_Entry, 0, len(current.dir)), - } - - for name, entry := range current.dir { - flushed, err := fs.flush(ctx, entry) - if err != nil { - return nil, err - } - - dir.Entries = append(dir.Entries, &protobuf.Directory_Entry{ - Name: name, - Ep: flushed.ep, - }) - } - - sort.Slice(dir.Entries, func(i, j int) bool { - return dir.Entries[i].Name < dir.Entries[j].Name - }) - - ep, err := fs.c.createProtobufMessage(ctx, blobtypes.Static, &dir) - if err != nil { - return nil, err - } - ep.ep.MimeType = CinodeDirMimeType - - return ep, nil + fs.rootEP = &nodeUnloaded{ep: *newRoot} + return nil } func (fs *cinodeFS) FindEntry(ctx context.Context, path []string) (*Entrypoint, error) { - return fs.findEntry(ctx, fs.rootEP, path, 0) -} - -func (fs *cinodeFS) findEntry( - ctx context.Context, - current *cachedEntrypoint, - path []string, - linkDepth int, -) (*Entrypoint, error) { - current, err := fs.loadEntrypoint(ctx, current) + var ret *Entrypoint + err := fs.traverseGraph( + ctx, + path, + traverseOptions{doNotCache: true}, + func(_ context.Context, ep node, _ bool) (node, dirtyState, error) { + var subErr error + ret, subErr = ep.entrypoint() + return nil, dsClean, subErr + }, + ) if err != nil { return nil, err } - - if current.link != nil { - if linkDepth >= fs.maxLinkRedirects { - return nil, ErrTooManyRedirects - } - return fs.findEntry(ctx, current.link.target, path, linkDepth+1) - } - - if current.dir != nil { - return fs.findEntryInDir(ctx, current.dir, path) - } - - if len(path) > 0 { - return nil, ErrNotADirectory - } - - return current.stored, nil -} - -func (fs *cinodeFS) findEntryInDir(ctx context.Context, dir dirCache, path []string) (*Entrypoint, error) { - if len(path) == 0 { - return nil, ErrModifiedDirectory - } - - entry, found := dir[path[0]] - if !found { - return nil, ErrEntryNotFound - } - - return fs.findEntry(ctx, entry, path[1:], 0) + return ret, nil } func (fs *cinodeFS) DeleteEntry(ctx context.Context, path []string) error { // Entry removal is done on the parent level, we find the parent directory // and remove the entry from its list - if len(path) == 0 { return ErrCantDeleteRoot } - newRoot, err := fs.deleteEntry( + return fs.traverseGraph( ctx, - fs.rootEP, path[:len(path)-1], - path[len(path)-1], - 0, + traverseOptions{createNodes: true}, + func(_ context.Context, reachedEntrypoint node, isWriteable bool) (node, dirtyState, error) { + if !isWriteable { + return nil, 0, ErrMissingWriterInfo + } + + dir, isDir := reachedEntrypoint.(*directoryNode) + if !isDir { + return nil, 0, ErrNotADirectory + } + + if !dir.deleteEntry(path[len(path)-1]) { + return nil, 0, ErrEntryNotFound + } + + return dir, dsDirty, nil + }, ) - if err != nil { - return err - } - fs.rootEP = newRoot - return nil } -func (fs *cinodeFS) deleteEntry( - ctx context.Context, - current *cachedEntrypoint, - path []string, - entryName string, - linkDepth int, -) ( - *cachedEntrypoint, - error, -) { - current, err := fs.loadEntrypoint(ctx, current) +func (fs *cinodeFS) GenerateNewDynamicLinkEntrypoint() (*Entrypoint, error) { + // Generate new entrypoint link data but do not yet store it in datastore + link, err := dynamiclink.Create(fs.randSource) if err != nil { return nil, err } - if current.link != nil { - if linkDepth >= fs.maxLinkRedirects { - return nil, ErrTooManyRedirects - } + bn := link.BlobName() + key := link.EncryptionKey() - if _, hasWriterInfo := fs.c.writerInfos[current.link.ep.BlobName().String()]; !hasWriterInfo { - // We won't be able to update data behind given link - // TODO: This is false for recursive links, we only have to check this at the last level - return nil, ErrMissingWriterInfo - } + fs.c.writerInfos[bn.String()] = link.AuthInfo() - newTarget, err := fs.deleteEntry( - ctx, - current.link.target, - path, - entryName, - linkDepth+1, - ) - if err != nil { - return nil, err - } - current.link.target = newTarget - return current, nil - } + return entrypointFromBlobNameAndKey(bn, key), nil +} - if current.dir == nil { - return nil, ErrNotADirectory - } +// func (fs *cinodeFS) ReplacePathWithLink(ctx context.Context, path []string) (WriterInfo, error) { - if len(path) == 0 { - // Got to the target directory, try to remove the entry - if _, found := current.dir[entryName]; !found { - return nil, ErrEntryNotFound - } - delete(current.dir, entryName) - return current, nil - } +// } - // Not yet at the target, descend to sub-directory - subDir, found := current.dir[path[0]] - if !found { - return nil, ErrEntryNotFound +func (fs *cinodeFS) OpenEntrypointData(ctx context.Context, ep *Entrypoint) (io.ReadCloser, error) { + if ep == nil { + return nil, ErrNilEntrypoint } - subDir, err = fs.deleteEntry(ctx, subDir, path[1:], entryName, 0) - if err != nil { - return nil, err - } + return fs.c.getDataReader(ctx, ep) +} - current.dir[path[0]] = subDir - return current, nil +func (fs *cinodeFS) RootEntrypoint() (*Entrypoint, error) { + return fs.rootEP.entrypoint() } -func (fs *cinodeFS) generateNewDynamicLinkEntrypoint() (*Entrypoint, error) { - // Generate new entrypoint link data but do not yet store it in datastore - link, err := dynamiclink.Create(fs.randSource) - if err != nil { - return nil, err +func (fs *cinodeFS) EntrypointWriterInfo(ctx context.Context, ep *Entrypoint) (WriterInfo, error) { + if !ep.IsLink() { + return WriterInfo{}, ErrNotALink } - bn := link.BlobName() - key := link.EncryptionKey() + bn := ep.BlobName() - fs.c.writerInfos[bn.String()] = link.AuthInfo() + key, err := fs.c.keyFromEntrypoint(ctx, ep) + if err != nil { + return WriterInfo{}, err + } - return entrypointFromBlobNameAndKey(bn, key), nil + authInfo, found := fs.c.writerInfos[bn.String()] + if !found { + return WriterInfo{}, ErrMissingWriterInfo + } + + return writerInfoFromBlobNameKeyAndAuthInfo(bn, key, authInfo), nil } -func (fs *cinodeFS) OpenEntrypointData(ctx context.Context, ep *Entrypoint) (io.ReadCloser, error) { - if ep == nil { - return nil, ErrNilEntrypoint +func (fs *cinodeFS) RootWriterInfo(ctx context.Context) (WriterInfo, error) { + rootEP, err := fs.RootEntrypoint() + if err != nil { + return WriterInfo{}, err } - return fs.c.getDataReader(ctx, ep) + return fs.EntrypointWriterInfo(ctx, rootEP) } diff --git a/pkg/structure/graph/cinodefs_blackbox_test.go b/pkg/structure/graph/cinodefs_blackbox_test.go index aeb568c..4470317 100644 --- a/pkg/structure/graph/cinodefs_blackbox_test.go +++ b/pkg/structure/graph/cinodefs_blackbox_test.go @@ -1,3 +1,19 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package graph_test import ( @@ -11,101 +27,320 @@ import ( "github.com/cinode/go/pkg/datastore" "github.com/cinode/go/pkg/structure/graph" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) -func TestSampleScenario(t *testing.T) { - ds := datastore.InMemory() - be := blenc.FromDatastore(ds) - +func TestCinodeFSSingleFileScenario(t *testing.T) { ctx := context.Background() - - fs, err := graph.NewCinodeFS( - ctx, - be, + fs, err := graph.NewCinodeFS(ctx, + blenc.FromDatastore(datastore.InMemory()), graph.NewRootDynamicLink(), ) require.NoError(t, err) require.NotNil(t, fs) - path1 := []string{"dir", "subdir", "file.txt"} + { // Check single file write operation + path1 := []string{"dir", "subdir", "file.txt"} - ep1, err := fs.SetEntryFile(ctx, - path1, - strings.NewReader("Hello world!"), - "", - ) - require.NoError(t, err) - require.NotNil(t, ep1) + ep1, err := fs.SetEntryFile(ctx, + path1, + strings.NewReader("Hello world!"), + ) + require.NoError(t, err) + require.NotNil(t, ep1) - ep2, err := fs.FindEntry( - ctx, - path1, - ) - require.NoError(t, err) - require.NotNil(t, ep2) + ep2, err := fs.FindEntry( + ctx, + path1, + ) + require.NoError(t, err) + require.NotNil(t, ep2) - require.Equal(t, ep1.String(), ep2.String()) + require.Equal(t, ep1.String(), ep2.String()) - // Get the entrypoint to the root - ep3, err := fs.FindEntry(ctx, []string{}) - require.ErrorIs(t, err, graph.ErrModifiedDirectory) - require.Nil(t, ep3) + // Directories are modified, not yet flushed + for i := range path1 { + ep3, err := fs.FindEntry(ctx, path1[:i]) + require.ErrorIs(t, err, graph.ErrModifiedDirectory) + require.Nil(t, ep3) + } - err = fs.Flush(ctx) - require.NoError(t, err) + err = fs.Flush(ctx) + require.NoError(t, err) + } +} + +type testFileEntry struct { + path []string + content string + mimeType string +} + +type CinodeFSMultiFileTestSuite struct { + suite.Suite + + ds datastore.DS + fs graph.CinodeFS + contentMap []testFileEntry +} - contentMap := make([]struct { - path []string - content string - }, 1000) +func TestCinodeFSMultiFileTestSuite(t *testing.T) { + suite.Run(t, &CinodeFSMultiFileTestSuite{}) +} + +func (c *CinodeFSMultiFileTestSuite) SetupTest() { + ctx := context.Background() + c.ds = datastore.InMemory() + fs, err := graph.NewCinodeFS(ctx, + blenc.FromDatastore(c.ds), + graph.NewRootDynamicLink(), + ) + require.NoError(c.T(), err) + require.NotNil(c.T(), fs) + c.fs = fs + + c.contentMap = make([]testFileEntry, 1000) for i := 0; i < 1000; i++ { - contentMap[i].path = []string{ + c.contentMap[i].path = []string{ fmt.Sprintf("dir%d", i%7), fmt.Sprintf("subdir%d", i%19), fmt.Sprintf("file%d.txt", i), } - contentMap[i].content = fmt.Sprintf("Hello world! from file %d!", i) + c.contentMap[i].content = fmt.Sprintf("Hello world! from file %d!", i) + c.contentMap[i].mimeType = "text/plain" } - for _, c := range contentMap { - _, err := fs.SetEntryFile(ctx, - c.path, - strings.NewReader(c.content), - "", + for _, file := range c.contentMap { + _, err := c.fs.SetEntryFile(ctx, + file.path, + strings.NewReader(file.content), ) - require.NoError(t, err) + require.NoError(c.T(), err) + } + + c.checkContentMap(fs) + + err = c.fs.Flush(context.Background()) + require.NoError(c.T(), err) + + c.checkContentMap(c.fs) +} + +func (c *CinodeFSMultiFileTestSuite) checkContentMap(fs graph.CinodeFS) { + ctx := context.Background() + for _, file := range c.contentMap { + ep, err := fs.FindEntry(ctx, file.path) + require.NoError(c.T(), err) + require.Contains(c.T(), ep.MimeType(), file.mimeType) + + rc, err := fs.OpenEntrypointData(ctx, ep) + require.NoError(c.T(), err) + defer rc.Close() + + data, err := io.ReadAll(rc) + require.NoError(c.T(), err) + + require.Equal(c.T(), file.content, string(data)) } +} + +func (c *CinodeFSMultiFileTestSuite) TestReopeningInReadOnlyMode() { + ctx := context.Background() + rootEP, err := c.fs.RootEntrypoint() + require.NoError(c.T(), err) + + fs2, err := graph.NewCinodeFS( + ctx, + blenc.FromDatastore(c.ds), + graph.RootEntrypoint(rootEP), + ) + require.NoError(c.T(), err) + require.NotNil(c.T(), fs2) + + c.checkContentMap(fs2) + + _, err = c.fs.SetEntryFile(ctx, + c.contentMap[0].path, + strings.NewReader("modified content"), + ) + require.NoError(c.T(), err) + + // Data in fs was not yet flushed to the datastore, fs2 should still refer to the old content + c.checkContentMap(fs2) + + err = c.fs.Flush(ctx) + require.NoError(c.T(), err) + + // Check with modified content map + c.contentMap[0].content = "modified content" + c.checkContentMap(fs2) + + // We should not be allowed to modify fs2 without writer info + ep, err := fs2.SetEntryFile(ctx, c.contentMap[0].path, strings.NewReader("should fail")) + require.ErrorIs(c.T(), err, graph.ErrMissingWriterInfo) + require.Nil(c.T(), ep) + c.checkContentMap(c.fs) + c.checkContentMap(fs2) +} - checkContentMap := func(fs graph.CinodeFS) { - for _, c := range contentMap { - ep, err := fs.FindEntry(ctx, c.path) - require.NoError(t, err) - require.Contains(t, ep.MimeType(), "text/plain") +func (c *CinodeFSMultiFileTestSuite) TestReopeningInReadWriteMode() { + ctx := context.Background() + + rootWriterInfo, err := c.fs.RootWriterInfo(ctx) + require.NoError(c.T(), err) + require.NotNil(c.T(), rootWriterInfo) + + fs3, err := graph.NewCinodeFS( + ctx, + blenc.FromDatastore(c.ds), + graph.RootWriterInfo(rootWriterInfo), + ) + require.NoError(c.T(), err) + require.NotNil(c.T(), fs3) + + c.checkContentMap(fs3) - rc, err := fs.OpenEntrypointData(ctx, ep) - require.NoError(t, err) - defer rc.Close() + // With a proper auth info we can modify files in the root path + ep, err := fs3.SetEntryFile(ctx, c.contentMap[0].path, strings.NewReader("modified through fs3")) + require.NoError(c.T(), err) + require.NotNil(c.T(), ep) + + c.contentMap[0].content = "modified through fs3" + c.checkContentMap(fs3) +} + +func (c *CinodeFSMultiFileTestSuite) TestRemovalOfAFile() { + ctx := context.Background() + + err := c.fs.DeleteEntry(ctx, c.contentMap[0].path) + require.NoError(c.T(), err) + + c.contentMap = c.contentMap[1:] + c.checkContentMap(c.fs) +} + +func (c *CinodeFSMultiFileTestSuite) TestRemovalOfADirectory() { + ctx := context.Background() - data, err := io.ReadAll(rc) - require.NoError(t, err) + removedPath := c.contentMap[0].path[:2] - require.Equal(t, c.content, string(data)) + err := c.fs.DeleteEntry(ctx, removedPath) + require.NoError(c.T(), err) + + filteredEntries := []testFileEntry{} + removed := 0 + for _, e := range c.contentMap { + if e.path[0] == removedPath[0] && e.path[1] == removedPath[1] { + continue } + + filteredEntries = append(filteredEntries, e) + removed++ } - checkContentMap(fs) + c.contentMap = filteredEntries + require.NotZero(c.T(), removed) - err = fs.Flush(ctx) - require.NoError(t, err) + c.checkContentMap(c.fs) + + err = c.fs.DeleteEntry(ctx, removedPath) + require.ErrorIs(c.T(), err, graph.ErrEntryNotFound) + + c.checkContentMap(c.fs) +} + +func (c *CinodeFSMultiFileTestSuite) TestDeleteTreatFileAsDirectory() { + ctx := context.Background() + + path := append(c.contentMap[0].path, "sub-file") + err := c.fs.DeleteEntry(ctx, path) + require.ErrorIs(c.T(), err, graph.ErrNotADirectory) +} + +func (c *CinodeFSMultiFileTestSuite) TestPreventSettingFileAsDirectory() { + ctx := context.Background() + + path := append(c.contentMap[0].path, "sub-file") + _, err := c.fs.SetEntryFile(ctx, path, strings.NewReader("should not happen")) + require.ErrorIs(c.T(), err, graph.ErrNotADirectory) +} - checkContentMap(fs) +func (c *CinodeFSMultiFileTestSuite) TestPreventSettingEmptyEntryName() { + ctx := context.Background() + + for _, path := range [][]string{ + {"", "subdir", "file.txt"}, + {"dir", "", "file.txt"}, + {"dir", "subdir", ""}, + } { + c.T().Run(strings.Join(path, "::"), func(t *testing.T) { + _, err := c.fs.SetEntryFile(ctx, path, strings.NewReader("should not succeed")) + require.ErrorIs(t, err, graph.ErrEmptyName) + + }) + } + +} + +func (c *CinodeFSMultiFileTestSuite) TestRootEPLinkOnDirtyFS() { + ctx := context.Background() + + ep1, err := c.fs.RootEntrypoint() + require.NoError(c.T(), err) + + _, err = c.fs.SetEntryFile(ctx, c.contentMap[0].path, strings.NewReader("hello")) + require.NoError(c.T(), err) + + ep2, err := c.fs.RootEntrypoint() + require.NoError(c.T(), err) + + // Even though dirty, entrypoint won't change it's content + require.Equal(c.T(), ep1.String(), ep2.String()) + + err = c.fs.Flush(ctx) + require.NoError(c.T(), err) + + ep3, err := c.fs.RootEntrypoint() + require.NoError(c.T(), err) + + require.Equal(c.T(), ep1.String(), ep3.String()) +} + +func (c *CinodeFSMultiFileTestSuite) TestRootEPDirectoryOnDirtyFS() { + ctx := context.Background() + + rootDir, err := c.fs.FindEntry(ctx, []string{}) + require.NoError(c.T(), err) + + fs2, err := graph.NewCinodeFS(ctx, + blenc.FromDatastore(c.ds), + graph.RootEntrypoint(rootDir), + ) + require.NoError(c.T(), err) + + ep1, err := fs2.RootEntrypoint() + require.NoError(c.T(), err) + require.Equal(c.T(), rootDir.String(), ep1.String()) + + _, err = fs2.SetEntryFile(ctx, c.contentMap[0].path, strings.NewReader("hello")) + require.NoError(c.T(), err) + + ep2, err := fs2.RootEntrypoint() + require.ErrorIs(c.T(), err, graph.ErrModifiedDirectory) + require.Nil(c.T(), ep2) + + err = fs2.Flush(ctx) + require.NoError(c.T(), err) + + ep3, err := c.fs.RootEntrypoint() + require.NoError(c.T(), err) + + require.NotEqual(c.T(), ep1.String(), ep3.String()) +} - // fs2, err := graph.NewCinodeFS( - // ctx, - // blenc.FromDatastore(ds), - // graph.RootEntrypointString() - // ) +func (c *CinodeFSMultiFileTestSuite) TestWriteOnlyLink() { + // ctx := context.Background() + // fs, err := graph.NewCinodeFS(ctx, blenc.FromDatastore(datastore.InMemory()), graph.NewRootDynamicLink()) + // require.NoError(c.T(), err) - // TODO: - // * reopening the datastore } diff --git a/pkg/structure/graph/cinodefs_options.go b/pkg/structure/graph/cinodefs_options.go index a86847b..97fd360 100644 --- a/pkg/structure/graph/cinodefs_options.go +++ b/pkg/structure/graph/cinodefs_options.go @@ -1,9 +1,27 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package graph import ( "context" "io" "time" + + "github.com/cinode/go/pkg/common" ) const ( @@ -29,21 +47,48 @@ func MaxLinkRedirects(maxLinkRedirects int) CinodeFSOption { func RootEntrypoint(ep *Entrypoint) CinodeFSOption { return optionFunc(func(ctx context.Context, fs *cinodeFS) error { - fs.rootEP = &cachedEntrypoint{stored: ep} + fs.rootEP = &nodeUnloaded{ep: *ep} return nil }) } +func errOption(err error) CinodeFSOption { + return optionFunc(func(ctx context.Context, fs *cinodeFS) error { return err }) +} + func RootEntrypointString(eps string) CinodeFSOption { ep, err := EntrypointFromString(eps) if err != nil { - return optionFunc(func(ctx context.Context, fs *cinodeFS) error { - return err - }) + return errOption(err) } return RootEntrypoint(ep) } +func RootWriterInfo(wi WriterInfo) CinodeFSOption { + bn, err := common.BlobNameFromBytes(wi.wi.BlobName) + if err != nil { + return errOption(err) + } + + key := common.BlobKeyFromBytes(wi.wi.Key) + ep := entrypointFromBlobNameAndKey(bn, key) + + return optionFunc(func(ctx context.Context, fs *cinodeFS) error { + fs.rootEP = &nodeUnloaded{ep: *ep} + fs.c.writerInfos[bn.String()] = wi.wi.AuthInfo + return nil + }) +} + +func RootWriterInfoString(wis string) CinodeFSOption { + wi, err := WriterInfoFromString(wis) + if err != nil { + return errOption(err) + } + + return RootWriterInfo(wi) +} + func TimeFunc(f func() time.Time) CinodeFSOption { return optionFunc(func(ctx context.Context, fs *cinodeFS) error { fs.timeFunc = f @@ -62,7 +107,7 @@ func RandSource(r io.Reader) CinodeFSOption { // dynamic link as the root func NewRootDynamicLink() CinodeFSOption { return optionFunc(func(ctx context.Context, fs *cinodeFS) error { - newLinkEntrypoint, err := fs.generateNewDynamicLinkEntrypoint() + newLinkEntrypoint, err := fs.GenerateNewDynamicLinkEntrypoint() if err != nil { return err } @@ -71,12 +116,11 @@ func NewRootDynamicLink() CinodeFSOption { // and an empty directory, all the entries are in-memory upon // creation and have to be flushed first to generate any // blobs - fs.rootEP = &cachedEntrypoint{ - link: &linkCache{ - ep: newLinkEntrypoint, - target: &cachedEntrypoint{ - dir: map[string]*cachedEntrypoint{}, - }, + fs.rootEP = &nodeLink{ + ep: *newLinkEntrypoint, + dState: dsSubDirty, + target: &directoryNode{ + entries: map[string]node{}, }, } return nil diff --git a/pkg/structure/graph/cinodefs_traverse.go b/pkg/structure/graph/cinodefs_traverse.go new file mode 100644 index 0000000..5ed03e4 --- /dev/null +++ b/pkg/structure/graph/cinodefs_traverse.go @@ -0,0 +1,70 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package graph + +import ( + "context" +) + +type traverseGoalFunc func( + ctx context.Context, + reachedEntrypoint node, + isWriteable bool, +) ( + replacementEntrypoint node, + changeResult dirtyState, + err error, +) + +type traverseOptions struct { + createNodes bool + doNotCache bool + maxLinkDepth int +} + +// Generic graph traversal function, it follows given path, once the endpoint +// is reached, it executed given callback function. +func (fs *cinodeFS) traverseGraph( + ctx context.Context, + path []string, + opts traverseOptions, + whenReached traverseGoalFunc, +) error { + for _, p := range path { + if p == "" { + return ErrEmptyName + } + } + + changedEntrypoint, _, err := fs.rootEP.traverse( + ctx, // context + &fs.c, // graph context + path, // path + 0, // pathPosition - start at the beginning + 0, // linkDepth - we don't come from any link + true, // isWritable - root is always writable + opts, // traverseOptions + whenReached, // callback + ) + if err != nil { + return err + } + if !opts.doNotCache { + fs.rootEP = changedEntrypoint + } + return nil +} diff --git a/pkg/structure/graph/context.go b/pkg/structure/graph/context.go index fc46b79..22240c5 100644 --- a/pkg/structure/graph/context.go +++ b/pkg/structure/graph/context.go @@ -1,3 +1,19 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package graph import ( @@ -9,6 +25,7 @@ import ( "github.com/cinode/go/pkg/blenc" "github.com/cinode/go/pkg/common" + "github.com/cinode/go/pkg/structure/internal/protobuf" "google.golang.org/protobuf/proto" ) @@ -108,7 +125,15 @@ func (c *graphContext) createProtobufMessage( c.writerInfos[bn.String()] = wi } - return entrypointFromBlobNameAndKey(bn, key), nil + return &Entrypoint{ + bn: bn, + ep: &protobuf.Entrypoint{ + BlobName: bn.Bytes(), + KeyInfo: &protobuf.KeyInfo{ + Key: key.Bytes(), + }, + }, + }, nil } func (c *graphContext) updateProtobufMessage( diff --git a/pkg/structure/graph/dir.go b/pkg/structure/graph/dir.go deleted file mode 100644 index ab66810..0000000 --- a/pkg/structure/graph/dir.go +++ /dev/null @@ -1,157 +0,0 @@ -package graph - -import ( - "context" - "errors" - "fmt" - "sort" - "strings" - - "github.com/cinode/go/pkg/blobtypes" - "github.com/cinode/go/pkg/structure/internal/protobuf" -) - -const ( - CinodeDirMimeType = "application/cinode-dir" -) - -var ( - // Returned when an entry does not exist in a directory - ErrEntryNotFound = errors.New("entry not found") - - // Returned when there's a directory read error - ErrCantReadDirectory = errors.New("can not read directory") - - // Returned when directory blob was read correctly but the data is corrupted - ErrInvalidDirectoryData = errors.New("invalid directory data") - - ErrCantWriteDirectory = errors.New("can not write directory") -) - -type Dir struct { - entries map[string]*Entrypoint -} - -func NewEmptyDir() *Dir { - return &Dir{ - entries: map[string]*Entrypoint{}, - } -} - -func LoadDir(ctx context.Context, ep *Entrypoint, c *graphContext) (*Dir, error) { - dirMessage := protobuf.Directory{} - - err := c.readProtobufMessage(ctx, ep, &dirMessage) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrInvalidDirectoryData, err) - } - - dir := &Dir{ - entries: make(map[string]*Entrypoint, len(dirMessage.Entries)), - } - - for _, entry := range dirMessage.Entries { - if entry.Name == "" { - return nil, fmt.Errorf("%w: entry with empty name", ErrInvalidDirectoryData) - } - ep, err := entrypointFromProtobuf(entry.Ep) - if err != nil { - return nil, fmt.Errorf( - "%w: invalid entrypoint for %s entry: %w", - ErrInvalidDirectoryData, entry.Name, err, - ) - } - if _, found := dir.entries[entry.Name]; found { - return nil, fmt.Errorf( - "%w: invalid entry for %s: duplicate found", - ErrInvalidDirectoryData, - entry.Name, - ) - } - dir.entries[entry.Name] = ep - } - - return dir, nil -} - -func (d *Dir) FindEntry(n string) (*Entrypoint, error) { - e, found := d.entries[n] - if !found { - return nil, ErrEntryNotFound - } - return e, nil -} - -type DirEntriesFilter struct { - NamePrefix string - NameSuffix string -} - -func (f *DirEntriesFilter) matches(name string, ep *Entrypoint) bool { - if f == nil { - return true - } - - if !strings.HasPrefix(name, f.NamePrefix) { - return false - } - if !strings.HasSuffix(name, f.NameSuffix) { - return false - } - - return true -} - -type DirEntriesFunc func(name string, ep *Entrypoint) - -func (d *Dir) EnumerateEntries( - ctx context.Context, - filter *DirEntriesFilter, - callback DirEntriesFunc, -) error { - for name, entry := range d.entries { - if ctx.Err() != nil { - return ctx.Err() - } - if filter.matches(name, entry) { - callback(name, entry) - } - } - return nil -} - -func (d *Dir) SetEntry(n string, ep *Entrypoint) { - d.entries[n] = ep -} - -func (d *Dir) DeleteEntry(n string) error { - if _, found := d.entries[n]; !found { - return ErrEntryNotFound - } - delete(d.entries, n) - return nil -} - -func (d *Dir) Store(ctx context.Context, c *graphContext) (*Entrypoint, error) { - dir := protobuf.Directory{ - Entries: make([]*protobuf.Directory_Entry, len(d.entries)), - } - - for name, entry := range d.entries { - dir.Entries = append(dir.Entries, &protobuf.Directory_Entry{ - Name: name, - Ep: entry.ep, - }) - } - - sort.Slice(dir.Entries, func(i, j int) bool { - return dir.Entries[i].Name < dir.Entries[j].Name - }) - - ep, err := c.createProtobufMessage(ctx, blobtypes.Static, &dir) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrCantWriteDirectory, err) - } - - return ep, nil -} diff --git a/pkg/structure/graph/entrypoint.go b/pkg/structure/graph/entrypoint.go index 02c96fc..995661e 100644 --- a/pkg/structure/graph/entrypoint.go +++ b/pkg/structure/graph/entrypoint.go @@ -1,3 +1,19 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package graph import ( @@ -75,13 +91,15 @@ func entrypointFromProtobuf(data *protobuf.Entrypoint) (*Entrypoint, error) { } func entrypointFromBlobNameAndKey(bn common.BlobName, key common.BlobKey) *Entrypoint { + return entrypointFromBlobNameKeyAndProtoEntrypoint(bn, key, &protobuf.Entrypoint{}) +} + +func entrypointFromBlobNameKeyAndProtoEntrypoint(bn common.BlobName, key common.BlobKey, protoEp *protobuf.Entrypoint) *Entrypoint { + protoEp.BlobName = bn.Bytes() + protoEp.KeyInfo = &protobuf.KeyInfo{Key: key.Bytes()} + return &Entrypoint{ - ep: &protobuf.Entrypoint{ - BlobName: bn.Bytes(), - KeyInfo: &protobuf.KeyInfo{ - Key: key.Bytes(), - }, - }, + ep: protoEp, bn: bn, } } diff --git a/pkg/structure/graph/entrypoint_options.go b/pkg/structure/graph/entrypoint_options.go new file mode 100644 index 0000000..32cac4e --- /dev/null +++ b/pkg/structure/graph/entrypoint_options.go @@ -0,0 +1,67 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package graph + +import ( + "context" + "time" + + "github.com/cinode/go/pkg/structure/internal/protobuf" +) + +type EntrypointOption interface { + apply(ctx context.Context, opts *entrypointOptions) error +} + +type entrypointOptionBasicFunc func(opts *entrypointOptions) + +func (ep entrypointOptionBasicFunc) apply(ctx context.Context, opts *entrypointOptions) error { + ep(opts) + return nil +} + +type entrypointOptions struct { + ep *protobuf.Entrypoint +} + +func SetMimeType(mimeType string) EntrypointOption { + return entrypointOptionBasicFunc(func(ep *entrypointOptions) { + ep.ep.MimeType = mimeType + }) +} + +func SetNotValidBefore(t time.Time) EntrypointOption { + return entrypointOptionBasicFunc(func(opts *entrypointOptions) { + opts.ep.NotValidBeforeUnixMicro = t.UnixMicro() + }) +} + +func SetNotValidAfter(t time.Time) EntrypointOption { + return entrypointOptionBasicFunc(func(opts *entrypointOptions) { + opts.ep.NotValidAfterUnixMicro = t.UnixMicro() + }) +} + +func protoEntrypointFromOptions(ctx context.Context, opts ...EntrypointOption) (*protobuf.Entrypoint, error) { + scratchpad := entrypointOptions{ep: &protobuf.Entrypoint{}} + for _, o := range opts { + if err := o.apply(ctx, &scratchpad); err != nil { + return nil, err + } + } + return scratchpad.ep, nil +} diff --git a/pkg/structure/graph/headwriter.go b/pkg/structure/graph/headwriter.go index 0e854b9..bbe8ab1 100644 --- a/pkg/structure/graph/headwriter.go +++ b/pkg/structure/graph/headwriter.go @@ -1,3 +1,19 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package graph type headWriter struct { diff --git a/pkg/structure/graph/link.go b/pkg/structure/graph/link.go deleted file mode 100644 index ff006fb..0000000 --- a/pkg/structure/graph/link.go +++ /dev/null @@ -1,66 +0,0 @@ -package graph - -import ( - "context" - "fmt" - - "github.com/cinode/go/pkg/blobtypes" - "github.com/cinode/go/pkg/structure/internal/protobuf" -) - -type Link struct { - ep *Entrypoint - tep *Entrypoint -} - -func NewLink( - ctx context.Context, - targetEP *Entrypoint, - c *graphContext, -) (*Link, error) { - ep, err := c.createProtobufMessage(ctx, blobtypes.DynamicLink, targetEP.ep) - if err != nil { - return nil, err - } - - return &Link{ - ep: ep, - tep: targetEP, - }, nil -} - -func OpenLink( - ctx context.Context, - ep *Entrypoint, - c *graphContext, -) (*Link, error) { - tepRaw := protobuf.Entrypoint{} - - err := c.readProtobufMessage(ctx, ep, &tepRaw) - if err != nil { - return nil, err - } - - tep, err := entrypointFromProtobuf(&tepRaw) - if err != nil { - return nil, err - } - - return &Link{ - ep: ep, - tep: tep, - }, nil -} - -func (l *Link) Update(ctx context.Context, tep *Entrypoint, c *graphContext) error { - err := c.updateProtobufMessage( - ctx, - l.ep, - tep.ep, - ) - if err != nil { - return fmt.Errorf("link update failed: %w", err) - } - l.tep = tep - return nil -} diff --git a/pkg/structure/graph/node.go b/pkg/structure/graph/node.go new file mode 100644 index 0000000..123de2e --- /dev/null +++ b/pkg/structure/graph/node.go @@ -0,0 +1,81 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package graph + +// +// cached entries: +// * unloaded entry - we only have entrypoint data +// * directory - either clean (with existing entrypoint) or dirty (modified entries, not yet flushed) +// * link - either clean (with tarted stored) or dirty (target changed but not yet flushed) +// * file - entrypoint to static blob +// +// node states: +// * if unloaded entry - contains entrypoint to the element, from entrypoint it can be deduced if this +// is a dynamic link (from blob name) or directory (from mime type), this node does not need flushing +// * node is dirty directly - the node was modified, its entrypoint can not be deduced before the node +// is flushed, some modifications are kept in memory and can still be lost +// * sub-nodes are dirty - the node itself is not dirty but some sub-nodes are. The node itself can have +// entrypoint deduced because it will not change, but some sub-nodes will need flushing to persist the +// data. Such situation is caused by dynamic links - the target can require flushing but the link itself +// will preserve its entrypoint. +// + +import ( + "context" +) + +type dirtyState byte + +const ( + // node and its sub-nodes are all clear, this sub-graph does not require flushing and is fully persisted + dsClean dirtyState = 0 + + // node is dirty, requires flushing to persist data + dsDirty dirtyState = 1 + + // node is itself clean, but some sub-nodes are dirty, flushing will be forwarded to sub-nodes + dsSubDirty dirtyState = 2 +) + +// node is a base interface required by all cached entries +type node interface { + // returns dirty state of this entrypoint + dirty() dirtyState + + // flush this entrypoint + flush(ctx context.Context, gc *graphContext) (*Entrypoint, error) + + // traverse node + traverse( + ctx context.Context, + gc *graphContext, + path []string, + pathPosition int, + linkDepth int, + isWritable bool, + opts traverseOptions, + whenReached traverseGoalFunc, + ) ( + replacementNode node, + state dirtyState, + err error, + ) + + // get current entrypoint value, do not flush before, if node is not flushed + // it must return appropriate error + entrypoint() (*Entrypoint, error) +} diff --git a/pkg/structure/graph/node_directory.go b/pkg/structure/graph/node_directory.go new file mode 100644 index 0000000..f7e8c71 --- /dev/null +++ b/pkg/structure/graph/node_directory.go @@ -0,0 +1,229 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package graph + +import ( + "context" + "sort" + + "github.com/cinode/go/pkg/blobtypes" + "github.com/cinode/go/pkg/structure/internal/protobuf" + "github.com/cinode/go/pkg/utilities/golang" +) + +// directoryNode holds a directory entry loaded into memory +type directoryNode struct { + entries map[string]node + stored *Entrypoint // current entrypoint, will be nil if directory was modified + dState dirtyState // true if any subtree is dirty +} + +func (d *directoryNode) dirty() dirtyState { + return d.dState +} + +func (d *directoryNode) flush(ctx context.Context, gc *graphContext) (*Entrypoint, error) { + if d.dState == dsClean { + // all clear, nothing to flush here or in sub-trees + return d.stored, nil + } + + if d.dState == dsSubDirty { + // Some sub-nodes are dirty, need to propagate flush to children + for _, entry := range d.entries { + if _, err := entry.flush(ctx, gc); err != nil { + return nil, err + } + } + + // directory itself was not modified and does not need flush, don't bother + // saving it to datastore + return d.stored, nil + } + + golang.Assert(d.dState == dsDirty, "ensure correct dirtiness state") + + // Directory has changed, have to recalculate its blob and save it in data store + dir := protobuf.Directory{ + Entries: make([]*protobuf.Directory_Entry, 0, len(d.entries)), + } + + for name, entry := range d.entries { + flushed, err := entry.flush(ctx, gc) + if err != nil { + return nil, err + } + + dir.Entries = append(dir.Entries, &protobuf.Directory_Entry{ + Name: name, + Ep: flushed.ep, + }) + } + + // Sort by name - that way we gain deterministic order during + // serialization od the directory + sort.Slice(dir.Entries, func(i, j int) bool { + return dir.Entries[i].Name < dir.Entries[j].Name + }) + + ep, err := gc.createProtobufMessage(ctx, blobtypes.Static, &dir) + if err != nil { + return nil, err + } + ep.ep.MimeType = CinodeDirMimeType + + return ep, nil +} + +func (c *directoryNode) traverse( + ctx context.Context, + gc *graphContext, + path []string, + pathPosition int, + linkDepth int, + isWritable bool, + opts traverseOptions, + whenReached traverseGoalFunc, +) ( + node, + dirtyState, + error, +) { + if pathPosition == len(path) { + return whenReached(ctx, c, isWritable) + } + + subNode, found := c.entries[path[pathPosition]] + if !found { + if !opts.createNodes { + return nil, 0, ErrEntryNotFound + } + if !isWritable { + return nil, 0, ErrMissingWriterInfo + } + // create new sub-path + newNode, err := c.traverseRecursiveNewPath( + ctx, + path, + pathPosition+1, + opts, + whenReached, + ) + if err != nil { + return nil, 0, err + } + c.entries[path[pathPosition]] = newNode + c.dState = dsDirty + return c, dsDirty, nil + } + + // found path entry, descend to sub-node + replacement, replacementState, err := subNode.traverse( + ctx, + gc, + path, + pathPosition+1, + 0, + isWritable, + opts, + whenReached, + ) + if err != nil { + return nil, 0, err + } + if opts.doNotCache { + return c, dsClean, nil + } + + c.entries[path[pathPosition]] = replacement + if replacementState == dsDirty { + // child is dirty, this propagates down to the current node + c.dState = dsDirty + return c, dsDirty, nil + } + + if replacementState == dsSubDirty { + // child itself is not dirty, but some sub-node is, sub-dirtiness + // propagates to the current node, but if the directory is + // already directly dirty (stronger dirtiness), keep it as it is + if c.dState != dsDirty { + c.dState = dsSubDirty + } + return c, dsSubDirty, nil + } + + golang.Assert(replacementState == dsClean, "ensure correct dirtiness state") + // leave current state as it is + return c, dsClean, nil + +} + +func (c *directoryNode) traverseRecursiveNewPath( + ctx context.Context, + path []string, + pathPosition int, + opts traverseOptions, + whenReached traverseGoalFunc, +) ( + node, + error, +) { + if len(path) == pathPosition { + replacement, _, err := whenReached(ctx, nil, true) + return replacement, err + } + + sub, err := c.traverseRecursiveNewPath( + ctx, + path, + pathPosition+1, + opts, + whenReached, + ) + if err != nil { + return nil, err + } + + return &directoryNode{ + entries: map[string]node{ + path[pathPosition]: sub, + }, + dState: dsDirty, + }, nil +} + +func (c *directoryNode) entrypoint() (*Entrypoint, error) { + if c.dState == dsDirty { + return nil, ErrModifiedDirectory + } + + golang.Assert( + c.dState == dsClean || c.dState == dsSubDirty, + "ensure dirtiness state is valid", + ) + + return c.stored, nil +} + +func (c *directoryNode) deleteEntry(name string) bool { + if _, hasEntry := c.entries[name]; !hasEntry { + return false + } + delete(c.entries, name) + c.dState = dsDirty + return true +} diff --git a/pkg/structure/graph/node_file.go b/pkg/structure/graph/node_file.go new file mode 100644 index 0000000..ce9d487 --- /dev/null +++ b/pkg/structure/graph/node_file.go @@ -0,0 +1,58 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package graph + +import "context" + +// Entry is a file with its entrypoint +type nodeFile struct { + ep Entrypoint +} + +func (c *nodeFile) dirty() dirtyState { + return dsClean +} + +func (c *nodeFile) flush(ctx context.Context, gc *graphContext) (*Entrypoint, error) { + return &c.ep, nil +} + +func (c *nodeFile) traverse( + ctx context.Context, + gc *graphContext, + path []string, + pathPosition int, + linkDepth int, + isWritable bool, + opts traverseOptions, + whenReached traverseGoalFunc, +) ( + node, + dirtyState, + error, +) { + if pathPosition == len(path) { + return whenReached(ctx, c, isWritable) + } + + // We're supposed to traverse into sub-path but it's not a directory + return nil, 0, ErrNotADirectory +} + +func (c *nodeFile) entrypoint() (*Entrypoint, error) { + return &c.ep, nil +} diff --git a/pkg/structure/graph/node_link.go b/pkg/structure/graph/node_link.go new file mode 100644 index 0000000..19d6b60 --- /dev/null +++ b/pkg/structure/graph/node_link.go @@ -0,0 +1,121 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package graph + +import ( + "context" + + "github.com/cinode/go/pkg/utilities/golang" +) + +// Entry is a link loaded into memory +type nodeLink struct { + ep Entrypoint // entrypoint of the link itself + target node // target for the link + dState dirtyState +} + +func (c *nodeLink) dirty() dirtyState { + return c.dState +} + +func (c *nodeLink) flush(ctx context.Context, gc *graphContext) (*Entrypoint, error) { + if c.dState == dsClean { + // all clear + return &c.ep, nil + } + + golang.Assert(c.dState == dsSubDirty, "link can be clean or sub-dirty") + target, err := c.target.flush(ctx, gc) + if err != nil { + return nil, err + } + + err = gc.updateProtobufMessage(ctx, &c.ep, target.ep) + if err != nil { + return nil, err + } + + return &c.ep, nil +} + +func (c *nodeLink) traverse( + ctx context.Context, + gc *graphContext, + path []string, + pathPosition int, + linkDepth int, + isWritable bool, + opts traverseOptions, + whenReached traverseGoalFunc, +) ( + node, + dirtyState, + error, +) { + if linkDepth > opts.maxLinkDepth { + return nil, 0, ErrTooManyRedirects + } + + // Note: we don't stop here even if we've reached the end of + // traverse path, delegate traversal to target node instead + + // crossing link border, whether sub-graph is writeable is determined + // by availability of corresponding writer info + _, hasWriterInfo := gc.writerInfos[c.ep.bn.String()] + + newTarget, targetState, err := c.target.traverse( + ctx, + gc, + path, + pathPosition, + linkDepth+1, + hasWriterInfo, + opts, + whenReached, + ) + if err != nil { + return nil, 0, err + } + + if opts.doNotCache { + return c, dsClean, nil + } + + c.target = newTarget + if targetState == dsClean { + // Nothing to do + // + // Note: this path will happen once we keep clean nodes + // in the memory for caching purposes + return c, dsClean, nil + } + + golang.Assert( + targetState == dsDirty || targetState == dsSubDirty, + "ensure correct dirtiness state", + ) + + // sub-dirty propagates normally, dirty becomes sub-dirty + // because link's entrypoint never changes + c.dState = dsSubDirty + return c, dsSubDirty, nil +} + +func (c *nodeLink) entrypoint() (*Entrypoint, error) { + return &c.ep, nil +} diff --git a/pkg/structure/graph/node_unloaded.go b/pkg/structure/graph/node_unloaded.go new file mode 100644 index 0000000..1e0f054 --- /dev/null +++ b/pkg/structure/graph/node_unloaded.go @@ -0,0 +1,135 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package graph + +import ( + "context" + "fmt" + + "github.com/cinode/go/pkg/structure/internal/protobuf" +) + +type nodeUnloaded struct { + ep Entrypoint +} + +func (c *nodeUnloaded) dirty() dirtyState { + return dsClean +} + +func (c *nodeUnloaded) flush(ctx context.Context, gc *graphContext) (*Entrypoint, error) { + return &c.ep, nil +} + +func (c *nodeUnloaded) traverse( + ctx context.Context, + gc *graphContext, + path []string, + pathPosition int, + linkDepth int, + isWritable bool, + opts traverseOptions, + whenReached traverseGoalFunc, +) ( + node, + dirtyState, + error, +) { + loaded, err := c.load(ctx, gc) + if err != nil { + return nil, 0, err + } + + return loaded.traverse( + ctx, + gc, + path, + pathPosition, + linkDepth, + isWritable, + opts, + whenReached, + ) +} + +func (c *nodeUnloaded) load(ctx context.Context, gc *graphContext) (node, error) { + // Data is behind some entrypoint, try to load it + if c.ep.IsLink() { + return c.loadEntrypointLink(ctx, gc) + } + + if c.ep.IsDir() { + return c.loadEntrypointDir(ctx, gc) + } + + return &nodeFile{ep: c.ep}, nil +} + +func (c *nodeUnloaded) loadEntrypointLink(ctx context.Context, gc *graphContext) (node, error) { + msg := &protobuf.Entrypoint{} + err := gc.readProtobufMessage(ctx, &c.ep, msg) + if err != nil { + return nil, err + } + + targetEP, err := entrypointFromProtobuf(msg) + if err != nil { + return nil, err + } + + return &nodeLink{ + ep: c.ep, + target: &nodeUnloaded{ep: *targetEP}, + dState: dsClean, + }, nil +} + +func (c *nodeUnloaded) loadEntrypointDir(ctx context.Context, gc *graphContext) (node, error) { + msg := &protobuf.Directory{} + err := gc.readProtobufMessage(ctx, &c.ep, msg) + if err != nil { + return nil, err + } + + dir := make(map[string]node, len(msg.Entries)) + + for _, entry := range msg.Entries { + if entry.Name == "" { + return nil, ErrEmptyName + } + if _, exists := dir[entry.Name]; exists { + return nil, fmt.Errorf("%w: %s", ErrDuplicateEntry, entry.Name) + } + + ep, err := entrypointFromProtobuf(entry.Ep) + if err != nil { + return nil, err + } + + dir[entry.Name] = &nodeUnloaded{ep: *ep} + } + + return &directoryNode{ + stored: &c.ep, + entries: dir, + dState: dsClean, + }, nil +} + +func (c *nodeUnloaded) entrypoint() (*Entrypoint, error) { + return &c.ep, nil +} diff --git a/pkg/structure/graph/writerinfo.go b/pkg/structure/graph/writerinfo.go index 69c5c0a..cba52a6 100644 --- a/pkg/structure/graph/writerinfo.go +++ b/pkg/structure/graph/writerinfo.go @@ -1,9 +1,26 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package graph import ( "errors" "fmt" + "github.com/cinode/go/pkg/common" "github.com/cinode/go/pkg/structure/internal/protobuf" "github.com/jbenet/go-base58" "google.golang.org/protobuf/proto" @@ -45,3 +62,13 @@ func WriterInfoFromBytes(b []byte) (WriterInfo, error) { func writerInfoFromProtobuf(data *protobuf.WriterInfo) (WriterInfo, error) { return WriterInfo{wi: data}, nil } + +func writerInfoFromBlobNameKeyAndAuthInfo(bn common.BlobName, key common.BlobKey, ai []byte) WriterInfo { + return WriterInfo{ + wi: &protobuf.WriterInfo{ + BlobName: bn.Bytes(), + Key: key.Bytes(), + AuthInfo: ai, + }, + } +} From d46cbc24d25a6aa94422529ccbd3829d72150251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Fri, 27 Oct 2023 12:27:53 +0200 Subject: [PATCH 08/29] Adopt existing code to use new CinodeFS traversal code --- pkg/cmd/cinode_web_proxy/root.go | 49 +++--- pkg/cmd/cinode_web_proxy/root_test.go | 50 +++--- pkg/cmd/static_datastore/compile.go | 90 +++++------ .../static_datastore/static_datastore_test.go | 40 +++-- pkg/structure/graph/cinodefs.go | 59 ++++++- pkg/structure/graph/cinodefs_options.go | 15 +- pkg/structure/graph/entrypoint.go | 2 +- pkg/structure/graph/writerinfo.go | 9 ++ pkg/structure/graphutils/directory.go | 153 ++++++++++++++++++ pkg/structure/graphutils/http.go | 105 ++++++++++++ testvectors/testblobs/base.go | 19 ++- testvectors/testblobs/dynamiclink.go | 13 +- 12 files changed, 483 insertions(+), 121 deletions(-) create mode 100644 pkg/structure/graphutils/directory.go create mode 100644 pkg/structure/graphutils/http.go diff --git a/pkg/cmd/cinode_web_proxy/root.go b/pkg/cmd/cinode_web_proxy/root.go index d8cf606..a3f4593 100644 --- a/pkg/cmd/cinode_web_proxy/root.go +++ b/pkg/cmd/cinode_web_proxy/root.go @@ -30,10 +30,9 @@ import ( "github.com/cinode/go/pkg/blenc" "github.com/cinode/go/pkg/datastore" - "github.com/cinode/go/pkg/protobuf" - "github.com/cinode/go/pkg/structure" + "github.com/cinode/go/pkg/structure/graph" + "github.com/cinode/go/pkg/structure/graphutils" "github.com/cinode/go/pkg/utilities/httpserver" - "github.com/jbenet/go-base58" "golang.org/x/exp/slog" ) @@ -60,13 +59,9 @@ func executeWithConfig(ctx context.Context, cfg *config) error { additionalDSs = append(additionalDSs, ds) } - entrypointRaw := base58.Decode(cfg.entrypoint) - if len(entrypointRaw) == 0 { - return errors.New("could not decode base58 entrypoint data") - } - entrypoint, err := protobuf.EntryPointFromBytes(entrypointRaw) + entrypoint, err := graph.EntrypointFromString(cfg.entrypoint) if err != nil { - return fmt.Errorf("could not unmarshal entrypoint data: %w", err) + return fmt.Errorf("could not parse entrypoint data: %w", err) } log := slog.Default() @@ -82,7 +77,11 @@ func executeWithConfig(ctx context.Context, cfg *config) error { "cpus", runtime.NumCPU(), ) - handler := setupCinodeProxy(mainDS, additionalDSs, entrypoint) + handler, err := setupCinodeProxy(ctx, mainDS, additionalDSs, entrypoint) + if err != nil { + return err + } + return httpserver.RunGracefully(ctx, handler, httpserver.ListenPort(cfg.port), @@ -91,22 +90,32 @@ func executeWithConfig(ctx context.Context, cfg *config) error { } func setupCinodeProxy( + ctx context.Context, mainDS datastore.DS, additionalDSs []datastore.DS, - entrypoint *protobuf.Entrypoint, -) http.Handler { - fs := structure.CinodeFS{ - BE: blenc.FromDatastore( - datastore.NewMultiSource(mainDS, time.Hour, additionalDSs...), + entrypoint *graph.Entrypoint, +) (http.Handler, error) { + fs, err := graph.NewCinodeFS( + ctx, + blenc.FromDatastore( + datastore.NewMultiSource( + mainDS, + time.Hour, + additionalDSs..., + ), ), - RootEntrypoint: entrypoint, - MaxLinkRedirects: 10, + graph.RootEntrypoint(entrypoint), + graph.MaxLinkRedirects(10), + ) + if err != nil { + return nil, err } - return &structure.HTTPHandler{ - FS: &fs, + return &graphutils.HTTPHandler{ + FS: fs, IndexFile: "index.html", - } + Log: slog.Default(), + }, nil } type config struct { diff --git a/pkg/cmd/cinode_web_proxy/root_test.go b/pkg/cmd/cinode_web_proxy/root_test.go index a6a9fe7..230e1cd 100644 --- a/pkg/cmd/cinode_web_proxy/root_test.go +++ b/pkg/cmd/cinode_web_proxy/root_test.go @@ -33,12 +33,11 @@ import ( "github.com/cinode/go/pkg/common" "github.com/cinode/go/pkg/datastore" "github.com/cinode/go/pkg/internal/utilities/cipherfactory" - "github.com/cinode/go/pkg/protobuf" - "github.com/cinode/go/pkg/structure" + "github.com/cinode/go/pkg/structure/graph" + "github.com/cinode/go/pkg/structure/graphutils" "github.com/cinode/go/testvectors/testblobs" "github.com/jbenet/go-base58" "github.com/stretchr/testify/require" - "golang.org/x/exp/slog" ) func TestGetConfig(t *testing.T) { @@ -111,17 +110,15 @@ func TestWebProxyHandlerInvalidEntrypoint(t *testing.T) { ) require.NoError(t, err) - handler := setupCinodeProxy( + key := cipherfactory.NewKeyGenerator(blobtypes.Static).Generate() + + handler, err := setupCinodeProxy( + context.Background(), datastore.InMemory(), []datastore.DS{}, - &protobuf.Entrypoint{ - BlobName: n.Bytes(), - MimeType: structure.CinodeDirMimeType, - KeyInfo: &protobuf.KeyInfo{ - Key: cipherfactory.NewKeyGenerator(blobtypes.Static).Generate().Bytes(), - }, - }, + graph.EntrypointFromBlobNameAndKey(n, key), ) + require.NoError(t, err) server := httptest.NewServer(handler) defer server.Close() @@ -148,7 +145,7 @@ func TestWebProxyHandlerSimplePage(t *testing.T) { ds := datastore.InMemory() be := blenc.FromDatastore(ds) - ep := func() *protobuf.Entrypoint { + ep := func() *graph.Entrypoint { dir := t.TempDir() for name, content := range map[string]string{ @@ -162,12 +159,23 @@ func TestWebProxyHandlerSimplePage(t *testing.T) { require.NoError(t, err) } - ep, err := structure.UploadStaticDirectory(context.Background(), slog.Default(), os.DirFS(dir), be) + fs, err := graph.NewCinodeFS(context.Background(), be, graph.NewRootDynamicLink()) + require.NoError(t, err) + + err = graphutils.UploadStaticDirectory( + context.Background(), + os.DirFS(dir), + fs, + ) + require.NoError(t, err) + + ep, err := fs.RootEntrypoint() require.NoError(t, err) return ep }() - handler := setupCinodeProxy(ds, []datastore.DS{}, ep) + handler, err := setupCinodeProxy(context.Background(), ds, []datastore.DS{}, ep) + require.NoError(t, err) server := httptest.NewServer(handler) defer server.Close() @@ -236,8 +244,7 @@ func TestExecuteWithConfig(t *testing.T) { }) t.Run("successful run", func(t *testing.T) { - epBytes, err := testblobs.DynamicLink.Entrypoint().ToBytes() - require.NoError(t, err) + ep := testblobs.DynamicLink.Entrypoint() ctx, cancel := context.WithCancel(context.Background()) go func() { @@ -245,9 +252,9 @@ func TestExecuteWithConfig(t *testing.T) { cancel() }() - err = executeWithConfig(ctx, &config{ + err := executeWithConfig(ctx, &config{ mainDSLocation: "memory://", - entrypoint: base58.Encode(epBytes), + entrypoint: ep.String(), }) require.NoError(t, err) }) @@ -255,16 +262,15 @@ func TestExecuteWithConfig(t *testing.T) { func TestExecute(t *testing.T) { t.Run("valid configuration", func(t *testing.T) { - epBytes, err := testblobs.DynamicLink.Entrypoint().ToBytes() - require.NoError(t, err) + ep := testblobs.DynamicLink.Entrypoint() - t.Setenv("CINODE_ENTRYPOINT", base58.Encode(epBytes)) + t.Setenv("CINODE_ENTRYPOINT", ep.String()) ctx, cancel := context.WithCancel(context.Background()) go func() { time.Sleep(10 * time.Millisecond) cancel() }() - err = Execute(ctx) + err := Execute(ctx) require.NoError(t, err) }) diff --git a/pkg/cmd/static_datastore/compile.go b/pkg/cmd/static_datastore/compile.go index ef2e6eb..05a13be 100644 --- a/pkg/cmd/static_datastore/compile.go +++ b/pkg/cmd/static_datastore/compile.go @@ -25,11 +25,9 @@ import ( "github.com/cinode/go/pkg/blenc" "github.com/cinode/go/pkg/datastore" - "github.com/cinode/go/pkg/protobuf" - "github.com/cinode/go/pkg/structure" - "github.com/jbenet/go-base58" + "github.com/cinode/go/pkg/structure/graph" + "github.com/cinode/go/pkg/structure/graphutils" "github.com/spf13/cobra" - "golang.org/x/exp/slog" ) func compileCmd() *cobra.Command { @@ -68,7 +66,7 @@ simple http server. log.Fatalf(msg) } - var wi *protobuf.WriterInfo + var wi *graph.WriterInfo if len(rootWriterInfoFile) > 0 { data, err := os.ReadFile(rootWriterInfoFile) if err != nil { @@ -80,39 +78,35 @@ simple http server. rootWriterInfoStr = string(data) } if len(rootWriterInfoStr) > 0 { - _wi, err := protobuf.WriterInfoFromBytes(base58.Decode(rootWriterInfoStr)) + _wi, err := graph.WriterInfoFromString(rootWriterInfoStr) if err != nil { fatalResult("Couldn't parse writer info: %v", err) } - wi = _wi + wi = &_wi } - ep, wi, err := compileFS(srcDir, dstDir, useStaticBlobs, wi, useRawFilesystem) + ep, wi, err := compileFS( + cmd.Context(), + srcDir, + dstDir, + useStaticBlobs, + wi, + useRawFilesystem, + ) if err != nil { fatalResult("%s", err) } - epBytes, err := ep.ToBytes() - if err != nil { - fatalResult("Couldn't serialize entrypoint: %v", err) - } - result := map[string]string{ "result": "OK", - "entrypoint": base58.Encode(epBytes), + "entrypoint": ep.String(), } if wi != nil { - wiBytes, err := wi.ToBytes() - if err != nil { - fatalResult("Couldn't serialize writer info: %v", err) - } - - result["writer-info"] = base58.Encode(wiBytes) + result["writer-info"] = wi.String() } enc.Encode(result) log.Println("DONE") - }, } @@ -127,17 +121,16 @@ simple http server. } func compileFS( + ctx context.Context, srcDir, dstDir string, static bool, - writerInfo *protobuf.WriterInfo, + writerInfo *graph.WriterInfo, useRawFS bool, ) ( - *protobuf.Entrypoint, - *protobuf.WriterInfo, + *graph.Entrypoint, + *graph.WriterInfo, error, ) { - var retWi *protobuf.WriterInfo - ds, err := func() (datastore.DS, error) { if useRawFS { return datastore.InRawFileSystem(dstDir) @@ -148,31 +141,38 @@ func compileFS( return nil, nil, fmt.Errorf("could not open datastore: %w", err) } - be := blenc.FromDatastore(ds) + opts := []graph.CinodeFSOption{} + if static { + opts = append(opts, graph.NewRootStaticDirectory()) + } else if writerInfo == nil { + opts = append(opts, graph.NewRootDynamicLink()) + } else { + opts = append(opts, graph.RootWriterInfo(*writerInfo)) + } - ep, err := structure.UploadStaticDirectory( - context.Background(), - slog.Default(), - os.DirFS(srcDir), - be, + fs, err := graph.NewCinodeFS( + ctx, + blenc.FromDatastore(ds), + opts..., ) + if err != nil { + return nil, nil, fmt.Errorf("couldn't create cinode filesystem instance: %w", err) + } + + err = graphutils.UploadStaticDirectory(ctx, os.DirFS(srcDir), fs) if err != nil { return nil, nil, fmt.Errorf("couldn't upload directory content: %w", err) } - if !static { - if writerInfo == nil { - ep, retWi, err = structure.CreateLink(context.Background(), be, ep) - if err != nil { - return nil, nil, fmt.Errorf("failed to update root link: %w", err) - } - } else { - ep, err = structure.UpdateLink(context.Background(), be, writerInfo, ep) - if err != nil { - return nil, nil, fmt.Errorf("failed to update root link: %w", err) - } - } + ep, err := fs.RootEntrypoint() + if err != nil { + return nil, nil, fmt.Errorf("couldn't get root entrypoint from cinodefs instance: %w", err) + } + + wi, err := fs.RootWriterInfo(ctx) + if err != nil { + return nil, nil, fmt.Errorf("couldn't get root writer info from cinodefs instance: %w", err) } - return ep, retWi, nil + return ep, &wi, nil } diff --git a/pkg/cmd/static_datastore/static_datastore_test.go b/pkg/cmd/static_datastore/static_datastore_test.go index e9b3855..2d6db88 100644 --- a/pkg/cmd/static_datastore/static_datastore_test.go +++ b/pkg/cmd/static_datastore/static_datastore_test.go @@ -18,6 +18,7 @@ package static_datastore import ( "bytes" + "context" "io" "net/http" "net/http/httptest" @@ -27,10 +28,11 @@ import ( "github.com/cinode/go/pkg/blenc" "github.com/cinode/go/pkg/datastore" - "github.com/cinode/go/pkg/protobuf" - "github.com/cinode/go/pkg/structure" + "github.com/cinode/go/pkg/structure/graph" + "github.com/cinode/go/pkg/structure/graphutils" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "golang.org/x/exp/slog" ) type datasetFile struct { @@ -100,10 +102,10 @@ func TestCompileAndReadTestSuite(t *testing.T) { func (s *CompileAndReadTestSuite) uploadDatasetToDatastore( dataset []datasetFile, datastoreDir string, - wi *protobuf.WriterInfo, -) (*protobuf.WriterInfo, *protobuf.Entrypoint) { + wi *graph.WriterInfo, +) (*graph.WriterInfo, *graph.Entrypoint) { - var ep *protobuf.Entrypoint + var ep *graph.Entrypoint s.T().Run("prepare dataset", func(t *testing.T) { dir := t.TempDir() @@ -116,7 +118,14 @@ func (s *CompileAndReadTestSuite) uploadDatasetToDatastore( s.Require().NoError(err) } - retEp, retWi, err := compileFS(dir, datastoreDir, false, wi, false) + retEp, retWi, err := compileFS( + context.Background(), + dir, + datastoreDir, + false, + wi, + false, + ) require.NoError(t, err) wi = retWi ep = retEp @@ -127,21 +136,24 @@ func (s *CompileAndReadTestSuite) uploadDatasetToDatastore( func (s *CompileAndReadTestSuite) validateDataset( dataset []datasetFile, - ep *protobuf.Entrypoint, + ep *graph.Entrypoint, datastoreDir string, ) { ds, err := datastore.InFileSystem(datastoreDir) s.Require().NoError(err) - fs := structure.CinodeFS{ - BE: blenc.FromDatastore(ds), - RootEntrypoint: ep, - MaxLinkRedirects: 10, - } + fs, err := graph.NewCinodeFS( + context.Background(), + blenc.FromDatastore(ds), + graph.RootEntrypoint(ep), + graph.MaxLinkRedirects(10), + ) + s.Require().NoError(err) - testServer := httptest.NewServer(&structure.HTTPHandler{ - FS: &fs, + testServer := httptest.NewServer(&graphutils.HTTPHandler{ + FS: fs, IndexFile: "index.html", + Log: slog.Default(), }) defer testServer.Close() diff --git a/pkg/structure/graph/cinodefs.go b/pkg/structure/graph/cinodefs.go index f505a46..58cf39d 100644 --- a/pkg/structure/graph/cinodefs.go +++ b/pkg/structure/graph/cinodefs.go @@ -54,6 +54,59 @@ const ( CinodeDirMimeType = "application/cinode-dir" ) +type CinodeFS interface { + SetEntryFile( + ctx context.Context, + path []string, + data io.Reader, + opts ...EntrypointOption, + ) (*Entrypoint, error) + + CreateFileEntrypoint( + ctx context.Context, + data io.Reader, + opts ...EntrypointOption, + ) (*Entrypoint, error) + + SetEntry( + ctx context.Context, + path []string, + ep *Entrypoint, + ) error + + Flush( + ctx context.Context, + ) error + + FindEntry( + ctx context.Context, + path []string, + ) (*Entrypoint, error) + + DeleteEntry( + ctx context.Context, + path []string, + ) error + + GenerateNewDynamicLinkEntrypoint() (*Entrypoint, error) + + OpenEntrypointData( + ctx context.Context, + ep *Entrypoint, + ) (io.ReadCloser, error) + + RootEntrypoint() (*Entrypoint, error) + + EntrypointWriterInfo( + ctx context.Context, + ep *Entrypoint, + ) (WriterInfo, error) + + RootWriterInfo( + ctx context.Context, + ) (WriterInfo, error) +} + type cinodeFS struct { c graphContext maxLinkRedirects int @@ -63,13 +116,11 @@ type cinodeFS struct { rootEP node } -type CinodeFS = *cinodeFS - func NewCinodeFS( ctx context.Context, be blenc.BE, options ...CinodeFSOption, -) (*cinodeFS, error) { +) (CinodeFS, error) { if be == nil { return nil, ErrInvalidBE } @@ -258,7 +309,7 @@ func (fs *cinodeFS) GenerateNewDynamicLinkEntrypoint() (*Entrypoint, error) { fs.c.writerInfos[bn.String()] = link.AuthInfo() - return entrypointFromBlobNameAndKey(bn, key), nil + return EntrypointFromBlobNameAndKey(bn, key), nil } // func (fs *cinodeFS) ReplacePathWithLink(ctx context.Context, path []string) (WriterInfo, error) { diff --git a/pkg/structure/graph/cinodefs_options.go b/pkg/structure/graph/cinodefs_options.go index 97fd360..3920e72 100644 --- a/pkg/structure/graph/cinodefs_options.go +++ b/pkg/structure/graph/cinodefs_options.go @@ -71,7 +71,7 @@ func RootWriterInfo(wi WriterInfo) CinodeFSOption { } key := common.BlobKeyFromBytes(wi.wi.Key) - ep := entrypointFromBlobNameAndKey(bn, key) + ep := EntrypointFromBlobNameAndKey(bn, key) return optionFunc(func(ctx context.Context, fs *cinodeFS) error { fs.rootEP = &nodeUnloaded{ep: *ep} @@ -121,8 +121,21 @@ func NewRootDynamicLink() CinodeFSOption { dState: dsSubDirty, target: &directoryNode{ entries: map[string]node{}, + dState: dsDirty, }, } return nil }) } + +// NewRootDynamicLink option can be used to create completely new, random +// dynamic link as the root +func NewRootStaticDirectory() CinodeFSOption { + return optionFunc(func(ctx context.Context, fs *cinodeFS) error { + fs.rootEP = &directoryNode{ + entries: map[string]node{}, + dState: dsDirty, + } + return nil + }) +} diff --git a/pkg/structure/graph/entrypoint.go b/pkg/structure/graph/entrypoint.go index 995661e..685a8f5 100644 --- a/pkg/structure/graph/entrypoint.go +++ b/pkg/structure/graph/entrypoint.go @@ -90,7 +90,7 @@ func entrypointFromProtobuf(data *protobuf.Entrypoint) (*Entrypoint, error) { return &ret, nil } -func entrypointFromBlobNameAndKey(bn common.BlobName, key common.BlobKey) *Entrypoint { +func EntrypointFromBlobNameAndKey(bn common.BlobName, key common.BlobKey) *Entrypoint { return entrypointFromBlobNameKeyAndProtoEntrypoint(bn, key, &protobuf.Entrypoint{}) } diff --git a/pkg/structure/graph/writerinfo.go b/pkg/structure/graph/writerinfo.go index cba52a6..7ccaada 100644 --- a/pkg/structure/graph/writerinfo.go +++ b/pkg/structure/graph/writerinfo.go @@ -22,6 +22,7 @@ import ( "github.com/cinode/go/pkg/common" "github.com/cinode/go/pkg/structure/internal/protobuf" + "github.com/cinode/go/pkg/utilities/golang" "github.com/jbenet/go-base58" "google.golang.org/protobuf/proto" ) @@ -35,6 +36,14 @@ type WriterInfo struct { wi *protobuf.WriterInfo } +func (wi *WriterInfo) Bytes() []byte { + return golang.Must(proto.Marshal(wi.wi)) +} + +func (wi *WriterInfo) String() string { + return base58.Encode(wi.Bytes()) +} + func WriterInfoFromString(s string) (WriterInfo, error) { if len(s) == 0 { return WriterInfo{}, fmt.Errorf("%w: empty string", ErrInvalidWriterInfoData) diff --git a/pkg/structure/graphutils/directory.go b/pkg/structure/graphutils/directory.go new file mode 100644 index 0000000..19f0f1f --- /dev/null +++ b/pkg/structure/graphutils/directory.go @@ -0,0 +1,153 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package graphutils + +import ( + "context" + "errors" + "fmt" + "io/fs" + "path" + + _ "embed" + + "github.com/cinode/go/pkg/blenc" + "github.com/cinode/go/pkg/structure/graph" + "golang.org/x/exp/slog" +) + +const ( + CinodeDirMimeType = "application/cinode-dir" +) + +var ( + ErrNotFound = blenc.ErrNotFound + ErrNotADirectory = errors.New("entry is not a directory") + ErrNotAFile = errors.New("entry is not a file") +) + +func UploadStaticDirectory( + ctx context.Context, + fsys fs.FS, + cfs graph.CinodeFS, + opts ...UploadStaticDirectoryOption, +) error { + c := dirCompiler{ + ctx: ctx, + fsys: fsys, + cfs: cfs, + log: slog.Default(), + } + for _, opt := range opts { + if err := opt(&c); err != nil { + return err + } + } + + err := c.compilePath(ctx, ".", c.basePath) + if err != nil { + return err + } + + err = cfs.Flush(ctx) + if err != nil { + return err + } + + return nil +} + +type UploadStaticDirectoryOption func(d *dirCompiler) error + +func BasePath(path []string) UploadStaticDirectoryOption { + return UploadStaticDirectoryOption(func(d *dirCompiler) error { + d.basePath = path + return nil + }) +} + +type dirCompiler struct { + ctx context.Context + fsys fs.FS + cfs graph.CinodeFS + log *slog.Logger + basePath []string +} + +func (d *dirCompiler) compilePath( + ctx context.Context, + srcPath string, + destPath []string, +) error { + st, err := fs.Stat(d.fsys, srcPath) + if err != nil { + d.log.ErrorCtx(ctx, "failed to stat path", "path", srcPath, "err", err) + return fmt.Errorf("couldn't check path: %w", err) + } + + if st.IsDir() { + return d.compileDir(ctx, srcPath, destPath) + } + + if st.Mode().IsRegular() { + return d.compileFile(ctx, srcPath, destPath) + } + + d.log.ErrorContext(ctx, "path is neither dir nor a regular file", "path", srcPath) + return fmt.Errorf("neither dir nor a regular file: %v", srcPath) +} + +func (d *dirCompiler) compileFile(ctx context.Context, srcPath string, dstPath []string) error { + d.log.InfoContext(ctx, "compiling file", "path", srcPath) + fl, err := d.fsys.Open(srcPath) + if err != nil { + d.log.ErrorContext(ctx, "failed to open file", "path", srcPath, "err", err) + return fmt.Errorf("couldn't open file %v: %w", srcPath, err) + } + defer fl.Close() + + _, err = d.cfs.SetEntryFile(ctx, dstPath, fl) + if err != nil { + return fmt.Errorf("failed to upload file %v: %w", srcPath, err) + } + + return nil +} + +func (d *dirCompiler) compileDir(ctx context.Context, srcPath string, dstPath []string) error { + fileList, err := fs.ReadDir(d.fsys, srcPath) + if err != nil { + d.log.ErrorContext(ctx, "couldn't read contents of dir", "path", srcPath, "err", err) + return fmt.Errorf("couldn't read contents of dir %v: %w", srcPath, err) + } + + // TODO: Reset directory content + // TODO: Build index file + + for _, e := range fileList { + err := d.compilePath( + ctx, + path.Join(srcPath, e.Name()), + append(dstPath, e.Name()), + ) + if err != nil { + return err + } + } + + return nil +} diff --git a/pkg/structure/graphutils/http.go b/pkg/structure/graphutils/http.go new file mode 100644 index 0000000..c59c10c --- /dev/null +++ b/pkg/structure/graphutils/http.go @@ -0,0 +1,105 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package graphutils + +import ( + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/cinode/go/pkg/structure/graph" + "golang.org/x/exp/slog" +) + +type HTTPHandler struct { + FS graph.CinodeFS + IndexFile string + Log *slog.Logger +} + +func (h *HTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + log := h.Log.With( + slog.String("RemoteAddr", r.RemoteAddr), + slog.String("URL", r.URL.String()), + slog.String("Method", r.Method), + ) + + if r.Method != "GET" { + log.Error("Method not allowed") + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + path := r.URL.Path + if strings.HasSuffix(path, "/") { + path += h.IndexFile + } + + pathList := strings.Split(strings.TrimPrefix(path, "/"), "/") + for i := range pathList { + p, err := url.PathUnescape(pathList[i]) + if err != nil { + log.WarnCtx(r.Context(), + "Incorrect request path", + "err", err, + ) + http.Error(w, + fmt.Sprintf("Could not unescape URL path segment: %s", err.Error()), + http.StatusBadRequest, + ) + return + } + pathList[i] = p + } + + fileEP, err := h.FS.FindEntry(r.Context(), pathList) + switch { + case errors.Is(err, graph.ErrEntryNotFound): + log.Warn("Not found") + http.NotFound(w, r) + return + case err != nil: + log.Error("Error serving request", "err", err) + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + + if fileEP.IsDir() { + http.Redirect(w, r, r.URL.Path+"/", http.StatusPermanentRedirect) + return + } + + w.Header().Set("Content-Type", fileEP.MimeType()) + rc, err := h.FS.OpenEntrypointData(r.Context(), fileEP) + if err != nil { + http.Error(w, + fmt.Sprintf("%s: %v", http.StatusText(http.StatusInternalServerError), err), + http.StatusInternalServerError, + ) + h.Log.Error("Error opening file", "err", err) + return + } + defer rc.Close() + + _, err = io.Copy(w, rc) + if err != nil { + h.Log.Error("Error sending file", "err", err) + } +} diff --git a/testvectors/testblobs/base.go b/testvectors/testblobs/base.go index df9f74e..89c641d 100644 --- a/testvectors/testblobs/base.go +++ b/testvectors/testblobs/base.go @@ -23,14 +23,15 @@ import ( "net/http" "net/url" - "github.com/cinode/go/pkg/structure/internal/protobuf" + "github.com/cinode/go/pkg/common" + "github.com/cinode/go/pkg/structure/graph" "github.com/jbenet/go-base58" ) type TestBlob struct { UpdateDataset []byte - BlobName []byte - EncryptionKey []byte + BlobName common.BlobName + EncryptionKey common.BlobKey DecryptedDataset []byte } @@ -98,11 +99,9 @@ func (s *TestBlob) Get(baseUrl string) ([]byte, error) { return body, nil } -func (s *TestBlob) Entrypoint() *protobuf.Entrypoint { - return &protobuf.Entrypoint{ - BlobName: s.BlobName, - KeyInfo: &protobuf.KeyInfo{ - Key: s.EncryptionKey, - }, - } +func (s *TestBlob) Entrypoint() *graph.Entrypoint { + return graph.EntrypointFromBlobNameAndKey( + s.BlobName, + s.EncryptionKey, + ) } diff --git a/testvectors/testblobs/dynamiclink.go b/testvectors/testblobs/dynamiclink.go index be15f2e..422d226 100644 --- a/testvectors/testblobs/dynamiclink.go +++ b/testvectors/testblobs/dynamiclink.go @@ -1,5 +1,10 @@ package testblobs +import ( + "github.com/cinode/go/pkg/common" + "github.com/cinode/go/pkg/utilities/golang" +) + var DynamicLink = TestBlob{ []byte{ 0x00, 0x11, 0xDA, 0xDD, 0x0F, 0x94, 0xBE, 0xCA, @@ -30,20 +35,20 @@ var DynamicLink = TestBlob{ 0xD8, 0x3F, 0xDD, 0xB1, 0x1F, 0x22, 0x7C, 0xD9, 0x73, 0xCA, 0x26, 0x11, 0x29, 0x79, 0x03, 0xF9, }, - []byte{ + golang.Must(common.BlobNameFromBytes([]byte{ 0x4F, 0xDA, 0x7E, 0xE6, 0xF2, 0x71, 0xB5, 0xEF, 0xFF, 0xD2, 0x05, 0x27, 0x0B, 0xBA, 0x11, 0x13, 0x13, 0xF5, 0xC9, 0x06, 0x9D, 0x6C, 0x36, 0x5F, 0x80, 0xD3, 0x50, 0xE3, 0xC5, 0x9B, 0x0E, 0x8D, 0xE6, - }, - []byte{ + })), + common.BlobKeyFromBytes([]byte{ 0x00, 0x79, 0xD2, 0x68, 0xC8, 0xEB, 0xD6, 0xA1, 0xBD, 0x5D, 0xE8, 0x63, 0x1C, 0xF7, 0x73, 0x73, 0x77, 0x26, 0x99, 0x4E, 0xC7, 0x35, 0xD9, 0x81, 0xB5, 0x20, 0xA8, 0xD7, 0xD9, 0x0B, 0xD5, 0x05, 0x49, - }, + }), []byte{ 0x64, 0x79, 0x6E, 0x61, 0x6D, 0x69, 0x63, 0x20, 0x6C, 0x69, 0x6E, 0x6B, From c9fd85052807f10e7cdc91418a501e41a5a7ce5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Fri, 27 Oct 2023 23:13:42 +0200 Subject: [PATCH 09/29] Prepare flushing algorithm for caching, fix test issues --- pkg/cmd/cinode_web_proxy/root_test.go | 15 +++++-- pkg/structure/graph/cinodefs.go | 10 +++-- pkg/structure/graph/cinodefs_blackbox_test.go | 8 ++++ pkg/structure/graph/node.go | 2 +- pkg/structure/graph/node_directory.go | 39 ++++++++++++------- pkg/structure/graph/node_file.go | 8 ++-- pkg/structure/graph/node_link.go | 20 ++++++---- pkg/structure/graph/node_unloaded.go | 4 +- pkg/structure/graphutils/http.go | 3 +- 9 files changed, 75 insertions(+), 34 deletions(-) diff --git a/pkg/cmd/cinode_web_proxy/root_test.go b/pkg/cmd/cinode_web_proxy/root_test.go index 230e1cd..ef1e07f 100644 --- a/pkg/cmd/cinode_web_proxy/root_test.go +++ b/pkg/cmd/cinode_web_proxy/root_test.go @@ -41,6 +41,8 @@ import ( ) func TestGetConfig(t *testing.T) { + os.Clearenv() + t.Run("default config", func(t *testing.T) { cfg, err := getConfig() require.ErrorContains(t, err, "ENTRYPOINT") @@ -232,7 +234,7 @@ func TestExecuteWithConfig(t *testing.T) { mainDSLocation: "memory://", entrypoint: "!@#$", }) - require.ErrorContains(t, err, "decode") + require.ErrorContains(t, err, "could not parse") }) t.Run("invalid entrypoint bytes", func(t *testing.T) { @@ -240,7 +242,7 @@ func TestExecuteWithConfig(t *testing.T) { mainDSLocation: "memory://", entrypoint: base58.Encode([]byte("1234567890")), }) - require.ErrorContains(t, err, "unmarshal") + require.ErrorContains(t, err, "could not parse") }) t.Run("successful run", func(t *testing.T) { @@ -261,6 +263,8 @@ func TestExecuteWithConfig(t *testing.T) { } func TestExecute(t *testing.T) { + os.Clearenv() + t.Run("valid configuration", func(t *testing.T) { ep := testblobs.DynamicLink.Entrypoint() @@ -275,7 +279,12 @@ func TestExecute(t *testing.T) { }) t.Run("invalid configuration", func(t *testing.T) { - err := Execute(context.Background()) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(10 * time.Millisecond) + cancel() + }() + err := Execute(ctx) require.ErrorContains(t, err, "CINODE_ENTRYPOINT") }) } diff --git a/pkg/structure/graph/cinodefs.go b/pkg/structure/graph/cinodefs.go index 58cf39d..931df0d 100644 --- a/pkg/structure/graph/cinodefs.go +++ b/pkg/structure/graph/cinodefs.go @@ -240,12 +240,12 @@ func (fs *cinodeFS) SetEntry( } func (fs *cinodeFS) Flush(ctx context.Context) error { - newRoot, err := fs.rootEP.flush(ctx, &fs.c) + _, newRootEP, err := fs.rootEP.flush(ctx, &fs.c) if err != nil { return err } - fs.rootEP = &nodeUnloaded{ep: *newRoot} + fs.rootEP = &nodeUnloaded{ep: *newRootEP} return nil } @@ -254,11 +254,13 @@ func (fs *cinodeFS) FindEntry(ctx context.Context, path []string) (*Entrypoint, err := fs.traverseGraph( ctx, path, - traverseOptions{doNotCache: true}, + traverseOptions{ + doNotCache: true, + }, func(_ context.Context, ep node, _ bool) (node, dirtyState, error) { var subErr error ret, subErr = ep.entrypoint() - return nil, dsClean, subErr + return ep, dsClean, subErr }, ) if err != nil { diff --git a/pkg/structure/graph/cinodefs_blackbox_test.go b/pkg/structure/graph/cinodefs_blackbox_test.go index 4470317..d3227d3 100644 --- a/pkg/structure/graph/cinodefs_blackbox_test.go +++ b/pkg/structure/graph/cinodefs_blackbox_test.go @@ -172,6 +172,14 @@ func (c *CinodeFSMultiFileTestSuite) TestReopeningInReadOnlyMode() { err = c.fs.Flush(ctx) require.NoError(c.T(), err) + // reopen fs2 to avoid any caching issues + fs2, err = graph.NewCinodeFS( + ctx, + blenc.FromDatastore(c.ds), + graph.RootEntrypoint(rootEP), + ) + require.NoError(c.T(), err) + // Check with modified content map c.contentMap[0].content = "modified content" c.checkContentMap(fs2) diff --git a/pkg/structure/graph/node.go b/pkg/structure/graph/node.go index 123de2e..d07a5ca 100644 --- a/pkg/structure/graph/node.go +++ b/pkg/structure/graph/node.go @@ -57,7 +57,7 @@ type node interface { dirty() dirtyState // flush this entrypoint - flush(ctx context.Context, gc *graphContext) (*Entrypoint, error) + flush(ctx context.Context, gc *graphContext) (node, *Entrypoint, error) // traverse node traverse( diff --git a/pkg/structure/graph/node_directory.go b/pkg/structure/graph/node_directory.go index f7e8c71..26342c2 100644 --- a/pkg/structure/graph/node_directory.go +++ b/pkg/structure/graph/node_directory.go @@ -36,23 +36,31 @@ func (d *directoryNode) dirty() dirtyState { return d.dState } -func (d *directoryNode) flush(ctx context.Context, gc *graphContext) (*Entrypoint, error) { +func (d *directoryNode) flush(ctx context.Context, gc *graphContext) (node, *Entrypoint, error) { if d.dState == dsClean { // all clear, nothing to flush here or in sub-trees - return d.stored, nil + return d, d.stored, nil } if d.dState == dsSubDirty { - // Some sub-nodes are dirty, need to propagate flush to children - for _, entry := range d.entries { - if _, err := entry.flush(ctx, gc); err != nil { - return nil, err + // Some sub-nodes are dirty, need to propagate flush to + flushedEntries := make(map[string]node, len(d.entries)) + for name, entry := range d.entries { + target, _, err := entry.flush(ctx, gc) + if err != nil { + return nil, nil, err } + + flushedEntries[name] = target } // directory itself was not modified and does not need flush, don't bother // saving it to datastore - return d.stored, nil + return &directoryNode{ + entries: flushedEntries, + stored: d.stored, + dState: dsClean, + }, d.stored, nil } golang.Assert(d.dState == dsDirty, "ensure correct dirtiness state") @@ -61,16 +69,17 @@ func (d *directoryNode) flush(ctx context.Context, gc *graphContext) (*Entrypoin dir := protobuf.Directory{ Entries: make([]*protobuf.Directory_Entry, 0, len(d.entries)), } - + flushedEntries := make(map[string]node, len(d.entries)) for name, entry := range d.entries { - flushed, err := entry.flush(ctx, gc) + target, targetEP, err := entry.flush(ctx, gc) if err != nil { - return nil, err + return nil, nil, err } + flushedEntries[name] = target dir.Entries = append(dir.Entries, &protobuf.Directory_Entry{ Name: name, - Ep: flushed.ep, + Ep: targetEP.ep, }) } @@ -82,11 +91,15 @@ func (d *directoryNode) flush(ctx context.Context, gc *graphContext) (*Entrypoin ep, err := gc.createProtobufMessage(ctx, blobtypes.Static, &dir) if err != nil { - return nil, err + return nil, nil, err } ep.ep.MimeType = CinodeDirMimeType - return ep, nil + return &directoryNode{ + entries: flushedEntries, + stored: ep, + dState: dsClean, + }, ep, nil } func (c *directoryNode) traverse( diff --git a/pkg/structure/graph/node_file.go b/pkg/structure/graph/node_file.go index ce9d487..6f98ea6 100644 --- a/pkg/structure/graph/node_file.go +++ b/pkg/structure/graph/node_file.go @@ -16,7 +16,9 @@ limitations under the License. package graph -import "context" +import ( + "context" +) // Entry is a file with its entrypoint type nodeFile struct { @@ -27,8 +29,8 @@ func (c *nodeFile) dirty() dirtyState { return dsClean } -func (c *nodeFile) flush(ctx context.Context, gc *graphContext) (*Entrypoint, error) { - return &c.ep, nil +func (c *nodeFile) flush(ctx context.Context, gc *graphContext) (node, *Entrypoint, error) { + return c, &c.ep, nil } func (c *nodeFile) traverse( diff --git a/pkg/structure/graph/node_link.go b/pkg/structure/graph/node_link.go index 19d6b60..b19cddf 100644 --- a/pkg/structure/graph/node_link.go +++ b/pkg/structure/graph/node_link.go @@ -33,24 +33,30 @@ func (c *nodeLink) dirty() dirtyState { return c.dState } -func (c *nodeLink) flush(ctx context.Context, gc *graphContext) (*Entrypoint, error) { +func (c *nodeLink) flush(ctx context.Context, gc *graphContext) (node, *Entrypoint, error) { if c.dState == dsClean { // all clear - return &c.ep, nil + return c, &c.ep, nil } golang.Assert(c.dState == dsSubDirty, "link can be clean or sub-dirty") - target, err := c.target.flush(ctx, gc) + target, targetEP, err := c.target.flush(ctx, gc) if err != nil { - return nil, err + return nil, nil, err } - err = gc.updateProtobufMessage(ctx, &c.ep, target.ep) + err = gc.updateProtobufMessage(ctx, &c.ep, targetEP.ep) if err != nil { - return nil, err + return nil, nil, err } - return &c.ep, nil + ret := &nodeLink{ + ep: c.ep, + target: target, + dState: dsClean, + } + + return ret, &ret.ep, nil } func (c *nodeLink) traverse( diff --git a/pkg/structure/graph/node_unloaded.go b/pkg/structure/graph/node_unloaded.go index 1e0f054..b09c0ce 100644 --- a/pkg/structure/graph/node_unloaded.go +++ b/pkg/structure/graph/node_unloaded.go @@ -31,8 +31,8 @@ func (c *nodeUnloaded) dirty() dirtyState { return dsClean } -func (c *nodeUnloaded) flush(ctx context.Context, gc *graphContext) (*Entrypoint, error) { - return &c.ep, nil +func (c *nodeUnloaded) flush(ctx context.Context, gc *graphContext) (node, *Entrypoint, error) { + return c, &c.ep, nil } func (c *nodeUnloaded) traverse( diff --git a/pkg/structure/graphutils/http.go b/pkg/structure/graphutils/http.go index c59c10c..f10ba91 100644 --- a/pkg/structure/graphutils/http.go +++ b/pkg/structure/graphutils/http.go @@ -71,7 +71,8 @@ func (h *HTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { fileEP, err := h.FS.FindEntry(r.Context(), pathList) switch { - case errors.Is(err, graph.ErrEntryNotFound): + case errors.Is(err, graph.ErrEntryNotFound), + errors.Is(err, graph.ErrNotADirectory): log.Warn("Not found") http.NotFound(w, r) return From 8e4cc2ca5faa65fed730815a0f7419da6cfe35d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Fri, 27 Oct 2023 23:18:18 +0200 Subject: [PATCH 10/29] Rename directoryNode->nodeDirectory to keep naming consistent --- pkg/cmd/cinode_web_proxy/root.go | 2 + pkg/cmd/static_datastore/compile.go | 16 ++- pkg/structure/graph/cinodefs.go | 33 +++++- pkg/structure/graph/cinodefs_options.go | 4 +- pkg/structure/graph/node_directory.go | 22 ++-- pkg/structure/graph/node_unloaded.go | 2 +- pkg/structure/graphutils/directory.go | 110 +++++++++++++++++--- pkg/structure/graphutils/templates/dir.html | 76 ++++++++++++++ 8 files changed, 232 insertions(+), 33 deletions(-) create mode 100644 pkg/structure/graphutils/templates/dir.html diff --git a/pkg/cmd/cinode_web_proxy/root.go b/pkg/cmd/cinode_web_proxy/root.go index a3f4593..8c587ec 100644 --- a/pkg/cmd/cinode_web_proxy/root.go +++ b/pkg/cmd/cinode_web_proxy/root.go @@ -69,6 +69,8 @@ func executeWithConfig(ctx context.Context, cfg *config) error { log.Info("Server listening for connections", "address", fmt.Sprintf("http://localhost:%d", cfg.port), ) + log.Info("Main datastore", "addr", cfg.mainDSLocation) + log.Info("Additional datastores", "addrs", cfg.additionalDSLocations) log.Info("System info", "goos", runtime.GOOS, diff --git a/pkg/cmd/static_datastore/compile.go b/pkg/cmd/static_datastore/compile.go index 05a13be..dbbd005 100644 --- a/pkg/cmd/static_datastore/compile.go +++ b/pkg/cmd/static_datastore/compile.go @@ -19,6 +19,7 @@ package static_datastore import ( "context" "encoding/json" + "errors" "fmt" "log" "os" @@ -159,7 +160,17 @@ func compileFS( return nil, nil, fmt.Errorf("couldn't create cinode filesystem instance: %w", err) } - err = graphutils.UploadStaticDirectory(ctx, os.DirFS(srcDir), fs) + err = fs.ResetDir(ctx, []string{}) + if err != nil { + return nil, nil, fmt.Errorf("failed to reset the root directory: %w", err) + } + + err = graphutils.UploadStaticDirectory( + ctx, + os.DirFS(srcDir), + fs, + graphutils.CreateIndexFile("index.html"), + ) if err != nil { return nil, nil, fmt.Errorf("couldn't upload directory content: %w", err) } @@ -170,6 +181,9 @@ func compileFS( } wi, err := fs.RootWriterInfo(ctx) + if errors.Is(err, graph.ErrNotALink) { + return ep, nil, nil + } if err != nil { return nil, nil, fmt.Errorf("couldn't get root writer info from cinodefs instance: %w", err) } diff --git a/pkg/structure/graph/cinodefs.go b/pkg/structure/graph/cinodefs.go index 931df0d..0a66192 100644 --- a/pkg/structure/graph/cinodefs.go +++ b/pkg/structure/graph/cinodefs.go @@ -74,6 +74,11 @@ type CinodeFS interface { ep *Entrypoint, ) error + ResetDir( + ctx context.Context, + path []string, + ) error + Flush( ctx context.Context, ) error @@ -239,6 +244,32 @@ func (fs *cinodeFS) SetEntry( ) } +func (fs *cinodeFS) ResetDir(ctx context.Context, path []string) error { + whenReached := func( + ctx context.Context, + current node, + isWriteable bool, + ) (node, dirtyState, error) { + if !isWriteable { + return nil, 0, ErrMissingWriterInfo + } + return &nodeDirectory{ + entries: map[string]node{}, + dState: dsDirty, + }, dsDirty, nil + } + + return fs.traverseGraph( + ctx, + path, + traverseOptions{ + createNodes: true, + maxLinkDepth: fs.maxLinkRedirects, + }, + whenReached, + ) +} + func (fs *cinodeFS) Flush(ctx context.Context) error { _, newRootEP, err := fs.rootEP.flush(ctx, &fs.c) if err != nil { @@ -285,7 +316,7 @@ func (fs *cinodeFS) DeleteEntry(ctx context.Context, path []string) error { return nil, 0, ErrMissingWriterInfo } - dir, isDir := reachedEntrypoint.(*directoryNode) + dir, isDir := reachedEntrypoint.(*nodeDirectory) if !isDir { return nil, 0, ErrNotADirectory } diff --git a/pkg/structure/graph/cinodefs_options.go b/pkg/structure/graph/cinodefs_options.go index 3920e72..cec7c2a 100644 --- a/pkg/structure/graph/cinodefs_options.go +++ b/pkg/structure/graph/cinodefs_options.go @@ -119,7 +119,7 @@ func NewRootDynamicLink() CinodeFSOption { fs.rootEP = &nodeLink{ ep: *newLinkEntrypoint, dState: dsSubDirty, - target: &directoryNode{ + target: &nodeDirectory{ entries: map[string]node{}, dState: dsDirty, }, @@ -132,7 +132,7 @@ func NewRootDynamicLink() CinodeFSOption { // dynamic link as the root func NewRootStaticDirectory() CinodeFSOption { return optionFunc(func(ctx context.Context, fs *cinodeFS) error { - fs.rootEP = &directoryNode{ + fs.rootEP = &nodeDirectory{ entries: map[string]node{}, dState: dsDirty, } diff --git a/pkg/structure/graph/node_directory.go b/pkg/structure/graph/node_directory.go index 26342c2..b66bc74 100644 --- a/pkg/structure/graph/node_directory.go +++ b/pkg/structure/graph/node_directory.go @@ -25,18 +25,18 @@ import ( "github.com/cinode/go/pkg/utilities/golang" ) -// directoryNode holds a directory entry loaded into memory -type directoryNode struct { +// nodeDirectory holds a directory entry loaded into memory +type nodeDirectory struct { entries map[string]node stored *Entrypoint // current entrypoint, will be nil if directory was modified dState dirtyState // true if any subtree is dirty } -func (d *directoryNode) dirty() dirtyState { +func (d *nodeDirectory) dirty() dirtyState { return d.dState } -func (d *directoryNode) flush(ctx context.Context, gc *graphContext) (node, *Entrypoint, error) { +func (d *nodeDirectory) flush(ctx context.Context, gc *graphContext) (node, *Entrypoint, error) { if d.dState == dsClean { // all clear, nothing to flush here or in sub-trees return d, d.stored, nil @@ -56,7 +56,7 @@ func (d *directoryNode) flush(ctx context.Context, gc *graphContext) (node, *Ent // directory itself was not modified and does not need flush, don't bother // saving it to datastore - return &directoryNode{ + return &nodeDirectory{ entries: flushedEntries, stored: d.stored, dState: dsClean, @@ -95,14 +95,14 @@ func (d *directoryNode) flush(ctx context.Context, gc *graphContext) (node, *Ent } ep.ep.MimeType = CinodeDirMimeType - return &directoryNode{ + return &nodeDirectory{ entries: flushedEntries, stored: ep, dState: dsClean, }, ep, nil } -func (c *directoryNode) traverse( +func (c *nodeDirectory) traverse( ctx context.Context, gc *graphContext, path []string, @@ -185,7 +185,7 @@ func (c *directoryNode) traverse( } -func (c *directoryNode) traverseRecursiveNewPath( +func (c *nodeDirectory) traverseRecursiveNewPath( ctx context.Context, path []string, pathPosition int, @@ -211,7 +211,7 @@ func (c *directoryNode) traverseRecursiveNewPath( return nil, err } - return &directoryNode{ + return &nodeDirectory{ entries: map[string]node{ path[pathPosition]: sub, }, @@ -219,7 +219,7 @@ func (c *directoryNode) traverseRecursiveNewPath( }, nil } -func (c *directoryNode) entrypoint() (*Entrypoint, error) { +func (c *nodeDirectory) entrypoint() (*Entrypoint, error) { if c.dState == dsDirty { return nil, ErrModifiedDirectory } @@ -232,7 +232,7 @@ func (c *directoryNode) entrypoint() (*Entrypoint, error) { return c.stored, nil } -func (c *directoryNode) deleteEntry(name string) bool { +func (c *nodeDirectory) deleteEntry(name string) bool { if _, hasEntry := c.entries[name]; !hasEntry { return false } diff --git a/pkg/structure/graph/node_unloaded.go b/pkg/structure/graph/node_unloaded.go index b09c0ce..94047a1 100644 --- a/pkg/structure/graph/node_unloaded.go +++ b/pkg/structure/graph/node_unloaded.go @@ -123,7 +123,7 @@ func (c *nodeUnloaded) loadEntrypointDir(ctx context.Context, gc *graphContext) dir[entry.Name] = &nodeUnloaded{ep: *ep} } - return &directoryNode{ + return &nodeDirectory{ stored: &c.ep, entries: dir, dState: dsClean, diff --git a/pkg/structure/graphutils/directory.go b/pkg/structure/graphutils/directory.go index 19f0f1f..ba4ed33 100644 --- a/pkg/structure/graphutils/directory.go +++ b/pkg/structure/graphutils/directory.go @@ -17,9 +17,11 @@ limitations under the License. package graphutils import ( + "bytes" "context" "errors" "fmt" + "html/template" "io/fs" "path" @@ -27,6 +29,7 @@ import ( "github.com/cinode/go/pkg/blenc" "github.com/cinode/go/pkg/structure/graph" + "github.com/cinode/go/pkg/utilities/golang" "golang.org/x/exp/slog" ) @@ -58,7 +61,7 @@ func UploadStaticDirectory( } } - err := c.compilePath(ctx, ".", c.basePath) + _, err := c.compilePath(ctx, ".", c.basePath) if err != nil { return err } @@ -80,52 +83,91 @@ func BasePath(path []string) UploadStaticDirectoryOption { }) } +func CreateIndexFile(indexFile string) UploadStaticDirectoryOption { + return UploadStaticDirectoryOption(func(d *dirCompiler) error { + d.createIndexFile = true + d.indexFileName = indexFile + return nil + }) +} + type dirCompiler struct { - ctx context.Context - fsys fs.FS - cfs graph.CinodeFS - log *slog.Logger - basePath []string + ctx context.Context + fsys fs.FS + cfs graph.CinodeFS + log *slog.Logger + basePath []string + createIndexFile bool + indexFileName string +} + +type dirEntry struct { + Name string + MimeType string + IsDir bool + Size int64 } func (d *dirCompiler) compilePath( ctx context.Context, srcPath string, destPath []string, -) error { +) (*dirEntry, error) { st, err := fs.Stat(d.fsys, srcPath) if err != nil { d.log.ErrorCtx(ctx, "failed to stat path", "path", srcPath, "err", err) - return fmt.Errorf("couldn't check path: %w", err) + return nil, fmt.Errorf("couldn't check path: %w", err) + } + + var name string + if len(destPath) > 0 { + name = destPath[len(destPath)-1] } if st.IsDir() { - return d.compileDir(ctx, srcPath, destPath) + err = d.compileDir(ctx, srcPath, destPath) + if err != nil { + return nil, err + } + return &dirEntry{ + Name: name, + MimeType: graph.CinodeDirMimeType, + IsDir: true, + }, nil } if st.Mode().IsRegular() { - return d.compileFile(ctx, srcPath, destPath) + mime, err := d.compileFile(ctx, srcPath, destPath) + if err != nil { + return nil, err + } + return &dirEntry{ + Name: name, + MimeType: mime, + IsDir: false, + Size: st.Size(), + }, nil } d.log.ErrorContext(ctx, "path is neither dir nor a regular file", "path", srcPath) - return fmt.Errorf("neither dir nor a regular file: %v", srcPath) + return nil, fmt.Errorf("neither dir nor a regular file: %v", srcPath) } -func (d *dirCompiler) compileFile(ctx context.Context, srcPath string, dstPath []string) error { +func (d *dirCompiler) compileFile(ctx context.Context, srcPath string, dstPath []string) (string, error) { d.log.InfoContext(ctx, "compiling file", "path", srcPath) fl, err := d.fsys.Open(srcPath) if err != nil { d.log.ErrorContext(ctx, "failed to open file", "path", srcPath, "err", err) - return fmt.Errorf("couldn't open file %v: %w", srcPath, err) + return "", fmt.Errorf("couldn't open file %v: %w", srcPath, err) } defer fl.Close() - _, err = d.cfs.SetEntryFile(ctx, dstPath, fl) + ep, err := d.cfs.SetEntryFile(ctx, dstPath, fl) if err != nil { - return fmt.Errorf("failed to upload file %v: %w", srcPath, err) + return "", fmt.Errorf("failed to upload file %v: %w", srcPath, err) } - return nil + return ep.MimeType(), nil } func (d *dirCompiler) compileDir(ctx context.Context, srcPath string, dstPath []string) error { @@ -138,8 +180,11 @@ func (d *dirCompiler) compileDir(ctx context.Context, srcPath string, dstPath [] // TODO: Reset directory content // TODO: Build index file + entries := make([]*dirEntry, 0, len(fileList)) + hasIndex := false + for _, e := range fileList { - err := d.compilePath( + entry, err := d.compilePath( ctx, path.Join(srcPath, e.Name()), append(dstPath, e.Name()), @@ -147,7 +192,38 @@ func (d *dirCompiler) compileDir(ctx context.Context, srcPath string, dstPath [] if err != nil { return err } + + if entry.Name == d.indexFileName { + hasIndex = true + } else { + entries = append(entries, entry) + } + } + + if d.createIndexFile && !hasIndex { + buf := bytes.NewBuffer(nil) + err = dirIndexTemplate.Execute(buf, map[string]any{ + "entries": entries, + "indexName": d.indexFileName, + }) + if err != nil { + return err + } + + _, err = d.cfs.SetEntryFile(ctx, + append(dstPath, d.indexFileName), + bytes.NewReader(buf.Bytes()), + ) + if err != nil { + return err + } } return nil } + +//go:embed templates/dir.html +var _dirIndexTemplateStr string +var dirIndexTemplate = golang.Must( + template.New("dir").Parse(_dirIndexTemplateStr), +) diff --git a/pkg/structure/graphutils/templates/dir.html b/pkg/structure/graphutils/templates/dir.html new file mode 100644 index 0000000..948d1d3 --- /dev/null +++ b/pkg/structure/graphutils/templates/dir.html @@ -0,0 +1,76 @@ +{{/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + + + + + Directory Listing + + + + +

Directory Listing

+ + + + + + + + {{range .entries}}{{if .IsDir}} + + + + + + + {{end}}{{end}} + {{range .entries}}{{if not .IsDir}} + + + + + + + {{end}}{{end}} +
DirNameSizeMimeType
[DIR]{{.Name}}{{.MimeType}}
{{.Name}}{{.Size}}{{.MimeType}}
+ + + From 731427584a756e7f086705e80e9697a1060a1329 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Sat, 28 Oct 2023 21:15:54 +0200 Subject: [PATCH 11/29] Enhance dir listing --- pkg/structure/graphutils/directory.go | 15 ++--- pkg/structure/graphutils/templates/dir.html | 67 ++++++++++++++++----- 2 files changed, 59 insertions(+), 23 deletions(-) diff --git a/pkg/structure/graphutils/directory.go b/pkg/structure/graphutils/directory.go index ba4ed33..9a95081 100644 --- a/pkg/structure/graphutils/directory.go +++ b/pkg/structure/graphutils/directory.go @@ -125,7 +125,7 @@ func (d *dirCompiler) compilePath( } if st.IsDir() { - err = d.compileDir(ctx, srcPath, destPath) + size, err := d.compileDir(ctx, srcPath, destPath) if err != nil { return nil, err } @@ -133,6 +133,7 @@ func (d *dirCompiler) compilePath( Name: name, MimeType: graph.CinodeDirMimeType, IsDir: true, + Size: int64(size), }, nil } @@ -170,11 +171,11 @@ func (d *dirCompiler) compileFile(ctx context.Context, srcPath string, dstPath [ return ep.MimeType(), nil } -func (d *dirCompiler) compileDir(ctx context.Context, srcPath string, dstPath []string) error { +func (d *dirCompiler) compileDir(ctx context.Context, srcPath string, dstPath []string) (int, error) { fileList, err := fs.ReadDir(d.fsys, srcPath) if err != nil { d.log.ErrorContext(ctx, "couldn't read contents of dir", "path", srcPath, "err", err) - return fmt.Errorf("couldn't read contents of dir %v: %w", srcPath, err) + return 0, fmt.Errorf("couldn't read contents of dir %v: %w", srcPath, err) } // TODO: Reset directory content @@ -190,7 +191,7 @@ func (d *dirCompiler) compileDir(ctx context.Context, srcPath string, dstPath [] append(dstPath, e.Name()), ) if err != nil { - return err + return 0, err } if entry.Name == d.indexFileName { @@ -207,7 +208,7 @@ func (d *dirCompiler) compileDir(ctx context.Context, srcPath string, dstPath [] "indexName": d.indexFileName, }) if err != nil { - return err + return 0, err } _, err = d.cfs.SetEntryFile(ctx, @@ -215,11 +216,11 @@ func (d *dirCompiler) compileDir(ctx context.Context, srcPath string, dstPath [] bytes.NewReader(buf.Bytes()), ) if err != nil { - return err + return 0, err } } - return nil + return len(fileList), nil } //go:embed templates/dir.html diff --git a/pkg/structure/graphutils/templates/dir.html b/pkg/structure/graphutils/templates/dir.html index 948d1d3..01e3ee4 100644 --- a/pkg/structure/graphutils/templates/dir.html +++ b/pkg/structure/graphutils/templates/dir.html @@ -26,12 +26,15 @@ table { width: 100%; border-collapse: collapse; + table-layout: fixed; + border: 1px solid #ddd; } th, td { padding: 8px; text-align: left; + border-right: 1px solid #ddd; } tr:nth-child(even) { @@ -42,6 +45,32 @@ background-color: #4CAF50; color: white; } + + th.is-dir { + width: 40px; + } + + th.name { + width: 60% + } + + th.size { + width: 100px; + } + + th.mimetype { + width: 40%; + } + + td.empty { + text-align: center; + font-style: italic; + } + + th.size, + td.size { + text-align: right; + } @@ -49,27 +78,33 @@

Directory Listing

- - - - + + + + + + {{- if eq (len .entries) 0 }} + + - {{range .entries}}{{if .IsDir}} + {{- else }} + {{- range .entries }}{{- if .IsDir }} - - - - + + + + - {{end}}{{end}} - {{range .entries}}{{if not .IsDir}} + {{- end }}{{- end }} + {{- range .entries }}{{- if not .IsDir }} - - - - + + + + - {{end}}{{end}} + {{- end }}{{- end }} + {{- end }}
DirNameSizeMimeTypeNameSizeMimeType
— Empty —
[DIR]{{.Name}}{{.MimeType}}[DIR]{{ .Name }}{{ .Size }} entries{{ .MimeType }}
{{.Name}}{{.Size}}{{.MimeType}}{{ .Name }}{{ .Size }} bytes{{ .MimeType }}
From 92ea4679254f55069796eb2035f25bb948a18d57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Sat, 28 Oct 2023 21:32:40 +0200 Subject: [PATCH 12/29] Better support for various destination locations in static datastore compiler. Now can use URL notation which allows direct upload to HTTP and HTTPS datastores. --- pkg/cmd/static_datastore/compile.go | 27 +++++++++---------- .../static_datastore/static_datastore_test.go | 1 - 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/pkg/cmd/static_datastore/compile.go b/pkg/cmd/static_datastore/compile.go index dbbd005..01353fa 100644 --- a/pkg/cmd/static_datastore/compile.go +++ b/pkg/cmd/static_datastore/compile.go @@ -33,14 +33,14 @@ import ( func compileCmd() *cobra.Command { - var srcDir, dstDir string + var srcDir, dstLocation string var useStaticBlobs bool var useRawFilesystem bool var rootWriterInfoStr string var rootWriterInfoFile string cmd := &cobra.Command{ - Use: "compile --source --destination ", + Use: "compile --source --destination ", Short: "Compile datastore from static files", Long: ` The compile command can be used to create an encrypted datastore from @@ -48,7 +48,7 @@ a content with static files that can then be used to serve through a simple http server. `, Run: func(cmd *cobra.Command, args []string) { - if srcDir == "" || dstDir == "" { + if srcDir == "" || dstLocation == "" { cmd.Help() return } @@ -86,13 +86,17 @@ simple http server. wi = &_wi } + if useRawFilesystem { + // For backwards compatibility + dstLocation = "file-raw://" + dstLocation + } + ep, wi, err := compileFS( cmd.Context(), srcDir, - dstDir, + dstLocation, useStaticBlobs, wi, - useRawFilesystem, ) if err != nil { fatalResult("%s", err) @@ -112,9 +116,10 @@ simple http server. } cmd.Flags().StringVarP(&srcDir, "source", "s", "", "Source directory with content to compile") - cmd.Flags().StringVarP(&dstDir, "destination", "d", "", "Destination directory for blobs") + cmd.Flags().StringVarP(&dstLocation, "destination", "d", "", "Location of destination datastore for blobs, can be a directory or an url prefixed with file://, file-raw://, http://, https://") cmd.Flags().BoolVarP(&useStaticBlobs, "static", "t", false, "If set to true, compile only the static dataset, do not create or update dynamic link") cmd.Flags().BoolVarP(&useRawFilesystem, "raw-filesystem", "r", false, "If set to true, use raw filesystem instead of the optimized one, can be used to create dataset for a standard http server") + cmd.Flags().MarkDeprecated("raw-filesystem", "use file-raw:// destination prefix instead") cmd.Flags().StringVarP(&rootWriterInfoStr, "writer-info", "w", "", "Writer info for the root dynamic link, if neither writer info nor writer info file is specified, a random writer info will be generated and printed out") cmd.Flags().StringVarP(&rootWriterInfoFile, "writer-info-file", "f", "", "Name of the file containing writer info for the root dynamic link, if neither writer info nor writer info file is specified, a random writer info will be generated and printed out") @@ -123,21 +128,15 @@ simple http server. func compileFS( ctx context.Context, - srcDir, dstDir string, + srcDir, dstLocation string, static bool, writerInfo *graph.WriterInfo, - useRawFS bool, ) ( *graph.Entrypoint, *graph.WriterInfo, error, ) { - ds, err := func() (datastore.DS, error) { - if useRawFS { - return datastore.InRawFileSystem(dstDir) - } - return datastore.InFileSystem(dstDir) - }() + ds, err := datastore.FromLocation(dstLocation) if err != nil { return nil, nil, fmt.Errorf("could not open datastore: %w", err) } diff --git a/pkg/cmd/static_datastore/static_datastore_test.go b/pkg/cmd/static_datastore/static_datastore_test.go index 2d6db88..3f9485f 100644 --- a/pkg/cmd/static_datastore/static_datastore_test.go +++ b/pkg/cmd/static_datastore/static_datastore_test.go @@ -124,7 +124,6 @@ func (s *CompileAndReadTestSuite) uploadDatasetToDatastore( datastoreDir, false, wi, - false, ) require.NoError(t, err) wi = retWi From 92ca3d5c9f41f48086433b33c42ac96debddc041 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Sat, 28 Oct 2023 22:08:01 +0200 Subject: [PATCH 13/29] Enhance datastore generator * Add option to generate index files * Add option to append to existing datastore * Small internal refactor --- pkg/cmd/public_node/root_test.go | 3 + pkg/cmd/static_datastore/compile.go | 125 ++++++++++++------ .../static_datastore/static_datastore_test.go | 12 +- 3 files changed, 93 insertions(+), 47 deletions(-) diff --git a/pkg/cmd/public_node/root_test.go b/pkg/cmd/public_node/root_test.go index a74a419..c9fee86 100644 --- a/pkg/cmd/public_node/root_test.go +++ b/pkg/cmd/public_node/root_test.go @@ -19,6 +19,7 @@ package public_node import ( "context" "net/http/httptest" + "os" "testing" "time" @@ -28,6 +29,8 @@ import ( ) func TestGetConfig(t *testing.T) { + os.Clearenv() + t.Run("default config", func(t *testing.T) { cfg := getConfig() require.Equal(t, "memory://", cfg.mainDSLocation) diff --git a/pkg/cmd/static_datastore/compile.go b/pkg/cmd/static_datastore/compile.go index 01353fa..2b0ce6a 100644 --- a/pkg/cmd/static_datastore/compile.go +++ b/pkg/cmd/static_datastore/compile.go @@ -23,6 +23,7 @@ import ( "fmt" "log" "os" + "strings" "github.com/cinode/go/pkg/blenc" "github.com/cinode/go/pkg/datastore" @@ -32,23 +33,21 @@ import ( ) func compileCmd() *cobra.Command { - - var srcDir, dstLocation string - var useStaticBlobs bool - var useRawFilesystem bool + var o compileFSOptions var rootWriterInfoStr string var rootWriterInfoFile string + var useRawFilesystem bool cmd := &cobra.Command{ Use: "compile --source --destination ", Short: "Compile datastore from static files", - Long: ` -The compile command can be used to create an encrypted datastore from -a content with static files that can then be used to serve through a -simple http server. -`, + Long: strings.Join([]string{ + "The compile command can be used to create an encrypted datastore from", + "a content with static files that can then be used to serve through a", + "simple http server.", + }, "\n"), Run: func(cmd *cobra.Command, args []string) { - if srcDir == "" || dstLocation == "" { + if o.srcDir == "" || o.dstLocation == "" { cmd.Help() return } @@ -67,7 +66,6 @@ simple http server. log.Fatalf(msg) } - var wi *graph.WriterInfo if len(rootWriterInfoFile) > 0 { data, err := os.ReadFile(rootWriterInfoFile) if err != nil { @@ -79,25 +77,19 @@ simple http server. rootWriterInfoStr = string(data) } if len(rootWriterInfoStr) > 0 { - _wi, err := graph.WriterInfoFromString(rootWriterInfoStr) + wi, err := graph.WriterInfoFromString(rootWriterInfoStr) if err != nil { fatalResult("Couldn't parse writer info: %v", err) } - wi = &_wi + o.writerInfo = &wi } if useRawFilesystem { // For backwards compatibility - dstLocation = "file-raw://" + dstLocation + o.dstLocation = "file-raw://" + o.dstLocation } - ep, wi, err := compileFS( - cmd.Context(), - srcDir, - dstLocation, - useStaticBlobs, - wi, - ) + ep, wi, err := compileFS(cmd.Context(), o) if err != nil { fatalResult("%s", err) } @@ -115,39 +107,85 @@ simple http server. }, } - cmd.Flags().StringVarP(&srcDir, "source", "s", "", "Source directory with content to compile") - cmd.Flags().StringVarP(&dstLocation, "destination", "d", "", "Location of destination datastore for blobs, can be a directory or an url prefixed with file://, file-raw://, http://, https://") - cmd.Flags().BoolVarP(&useStaticBlobs, "static", "t", false, "If set to true, compile only the static dataset, do not create or update dynamic link") - cmd.Flags().BoolVarP(&useRawFilesystem, "raw-filesystem", "r", false, "If set to true, use raw filesystem instead of the optimized one, can be used to create dataset for a standard http server") - cmd.Flags().MarkDeprecated("raw-filesystem", "use file-raw:// destination prefix instead") - cmd.Flags().StringVarP(&rootWriterInfoStr, "writer-info", "w", "", "Writer info for the root dynamic link, if neither writer info nor writer info file is specified, a random writer info will be generated and printed out") - cmd.Flags().StringVarP(&rootWriterInfoFile, "writer-info-file", "f", "", "Name of the file containing writer info for the root dynamic link, if neither writer info nor writer info file is specified, a random writer info will be generated and printed out") + cmd.Flags().StringVarP( + &o.srcDir, "source", "s", "", + "Source directory with content to compile", + ) + cmd.Flags().StringVarP( + &o.dstLocation, "destination", "d", "", + "location of destination datastore for blobs, can be a directory "+ + "or an url prefixed with file://, file-raw://, http://, https://", + ) + cmd.Flags().BoolVarP( + &o.static, "static", "t", false, + "if set to true, compile only the static dataset, do not create or update dynamic link", + ) + cmd.Flags().BoolVarP( + &useRawFilesystem, "raw-filesystem", "r", false, + "if set to true, use raw filesystem instead of the optimized one, "+ + "can be used to create dataset for a standard http server", + ) + cmd.Flags().MarkDeprecated( + "raw-filesystem", + "use file-raw:// destination prefix instead", + ) + cmd.Flags().StringVarP( + &rootWriterInfoStr, "writer-info", "w", "", + "writer info for the root dynamic link, if neither writer info nor writer info file is specified, "+ + "a random writer info will be generated and printed out", + ) + cmd.Flags().StringVarP( + &rootWriterInfoFile, "writer-info-file", "f", "", + "name of the file containing writer info for the root dynamic link, "+ + "if neither writer info nor writer info file is specified, "+ + "a random writer info will be generated and printed out", + ) + cmd.Flags().StringVar( + &o.indexFile, "index-file", "index.html", + "name of the index file", + ) + cmd.Flags().BoolVar( + &o.generateIndexFiles, "generate-index-files", false, + "automatically generate index html files with directory listing if index file is not present", + ) + cmd.Flags().BoolVar( + &o.append, "append", false, + "append file in existing datastore leaving existing unchanged files as is", + ) return cmd } +type compileFSOptions struct { + srcDir string + dstLocation string + static bool + writerInfo *graph.WriterInfo + generateIndexFiles bool + indexFile string + append bool +} + func compileFS( ctx context.Context, - srcDir, dstLocation string, - static bool, - writerInfo *graph.WriterInfo, + o compileFSOptions, ) ( *graph.Entrypoint, *graph.WriterInfo, error, ) { - ds, err := datastore.FromLocation(dstLocation) + ds, err := datastore.FromLocation(o.dstLocation) if err != nil { return nil, nil, fmt.Errorf("could not open datastore: %w", err) } opts := []graph.CinodeFSOption{} - if static { + if o.static { opts = append(opts, graph.NewRootStaticDirectory()) - } else if writerInfo == nil { + } else if o.writerInfo == nil { opts = append(opts, graph.NewRootDynamicLink()) } else { - opts = append(opts, graph.RootWriterInfo(*writerInfo)) + opts = append(opts, graph.RootWriterInfo(*o.writerInfo)) } fs, err := graph.NewCinodeFS( @@ -159,16 +197,23 @@ func compileFS( return nil, nil, fmt.Errorf("couldn't create cinode filesystem instance: %w", err) } - err = fs.ResetDir(ctx, []string{}) - if err != nil { - return nil, nil, fmt.Errorf("failed to reset the root directory: %w", err) + if !o.append { + err = fs.ResetDir(ctx, []string{}) + if err != nil { + return nil, nil, fmt.Errorf("failed to reset the root directory: %w", err) + } + } + + var genOpts []graphutils.UploadStaticDirectoryOption + if o.generateIndexFiles { + genOpts = append(genOpts, graphutils.CreateIndexFile(o.indexFile)) } err = graphutils.UploadStaticDirectory( ctx, - os.DirFS(srcDir), + os.DirFS(o.srcDir), fs, - graphutils.CreateIndexFile("index.html"), + genOpts..., ) if err != nil { return nil, nil, fmt.Errorf("couldn't upload directory content: %w", err) diff --git a/pkg/cmd/static_datastore/static_datastore_test.go b/pkg/cmd/static_datastore/static_datastore_test.go index 3f9485f..22c053c 100644 --- a/pkg/cmd/static_datastore/static_datastore_test.go +++ b/pkg/cmd/static_datastore/static_datastore_test.go @@ -118,13 +118,11 @@ func (s *CompileAndReadTestSuite) uploadDatasetToDatastore( s.Require().NoError(err) } - retEp, retWi, err := compileFS( - context.Background(), - dir, - datastoreDir, - false, - wi, - ) + retEp, retWi, err := compileFS(context.Background(), compileFSOptions{ + srcDir: dir, + dstLocation: datastoreDir, + writerInfo: wi, + }) require.NoError(t, err) wi = retWi ep = retEp From b700a073869c5d614bb922aabd0943f6f7d83a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Sat, 28 Oct 2023 22:38:47 +0200 Subject: [PATCH 14/29] Internal package structures refactoring --- pkg/{structure/graph => cinodefs}/cinodefs.go | 57 ++-- .../cinodefs_blackbox_test.go | 46 +-- .../graph => cinodefs}/cinodefs_options.go | 30 +- .../graph => cinodefs}/cinodefs_traverse.go | 2 +- pkg/{structure/graph => cinodefs}/context.go | 9 +- .../graph => cinodefs}/entrypoint.go | 56 ++-- .../graph => cinodefs}/entrypoint_options.go | 34 +-- .../graph => cinodefs}/headwriter.go | 2 +- .../httphandler}/http.go | 14 +- .../internal/protobuf/protobuf.go | 0 .../internal/protobuf/protobuf.pb.go | 0 .../internal/protobuf/protobuf.proto | 0 pkg/{structure/graph => cinodefs}/node.go | 2 +- .../graph => cinodefs}/node_directory.go | 6 +- .../graph => cinodefs}/node_file.go | 8 +- .../graph => cinodefs}/node_link.go | 14 +- .../graph => cinodefs}/node_unloaded.go | 24 +- .../uploader}/directory.go | 22 +- .../uploader}/templates/dir.html | 0 .../graph => cinodefs}/writerinfo.go | 34 +-- pkg/cmd/cinode_web_proxy/integration_test.go | 27 +- pkg/cmd/cinode_web_proxy/root.go | 16 +- pkg/cmd/cinode_web_proxy/root_test.go | 12 +- pkg/cmd/static_datastore/compile.go | 34 +-- .../static_datastore/static_datastore_test.go | 20 +- pkg/structure/cinodefs.go | 122 -------- pkg/structure/directory.go | 280 ------------------ pkg/structure/http.go | 72 ----- pkg/structure/link.go | 138 --------- pkg/structure/templates/dir.html | 73 ----- testvectors/testblobs/base.go | 6 +- 31 files changed, 242 insertions(+), 918 deletions(-) rename pkg/{structure/graph => cinodefs}/cinodefs.go (86%) rename pkg/{structure/graph => cinodefs}/cinodefs_blackbox_test.go (89%) rename pkg/{structure/graph => cinodefs}/cinodefs_options.go (81%) rename pkg/{structure/graph => cinodefs}/cinodefs_traverse.go (98%) rename pkg/{structure/graph => cinodefs}/context.go (95%) rename pkg/{structure/graph => cinodefs}/entrypoint.go (76%) rename pkg/{structure/graph => cinodefs}/entrypoint_options.go (50%) rename pkg/{structure/graph => cinodefs}/headwriter.go (98%) rename pkg/{structure/graphutils => cinodefs/httphandler}/http.go (89%) rename pkg/{structure => cinodefs}/internal/protobuf/protobuf.go (100%) rename pkg/{structure => cinodefs}/internal/protobuf/protobuf.pb.go (100%) rename pkg/{structure => cinodefs}/internal/protobuf/protobuf.proto (100%) rename pkg/{structure/graph => cinodefs}/node.go (99%) rename pkg/{structure/graph => cinodefs}/node_directory.go (98%) rename pkg/{structure/graph => cinodefs}/node_file.go (94%) rename pkg/{structure/graph => cinodefs}/node_link.go (91%) rename pkg/{structure/graph => cinodefs}/node_unloaded.go (85%) rename pkg/{structure/graphutils => cinodefs/uploader}/directory.go (90%) rename pkg/{structure/graphutils => cinodefs/uploader}/templates/dir.html (100%) rename pkg/{structure/graph => cinodefs}/writerinfo.go (62%) delete mode 100644 pkg/structure/cinodefs.go delete mode 100644 pkg/structure/directory.go delete mode 100644 pkg/structure/http.go delete mode 100644 pkg/structure/link.go delete mode 100644 pkg/structure/templates/dir.html diff --git a/pkg/structure/graph/cinodefs.go b/pkg/cinodefs/cinodefs.go similarity index 86% rename from pkg/structure/graph/cinodefs.go rename to pkg/cinodefs/cinodefs.go index 0a66192..5ed5b53 100644 --- a/pkg/structure/graph/cinodefs.go +++ b/pkg/cinodefs/cinodefs.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package graph +package cinodefs import ( "context" @@ -29,7 +29,6 @@ import ( "github.com/cinode/go/pkg/blenc" "github.com/cinode/go/pkg/blobtypes" "github.com/cinode/go/pkg/internal/blobtypes/dynamiclink" - "github.com/cinode/go/pkg/structure/internal/protobuf" ) var ( @@ -54,7 +53,7 @@ const ( CinodeDirMimeType = "application/cinode-dir" ) -type CinodeFS interface { +type FS interface { SetEntryFile( ctx context.Context, path []string, @@ -93,7 +92,10 @@ type CinodeFS interface { path []string, ) error - GenerateNewDynamicLinkEntrypoint() (*Entrypoint, error) + GenerateNewDynamicLinkEntrypoint() ( + *Entrypoint, + error, + ) OpenEntrypointData( ctx context.Context, @@ -105,11 +107,11 @@ type CinodeFS interface { EntrypointWriterInfo( ctx context.Context, ep *Entrypoint, - ) (WriterInfo, error) + ) (*WriterInfo, error) RootWriterInfo( ctx context.Context, - ) (WriterInfo, error) + ) (*WriterInfo, error) } type cinodeFS struct { @@ -121,11 +123,11 @@ type cinodeFS struct { rootEP node } -func NewCinodeFS( +func New( ctx context.Context, be blenc.BE, - options ...CinodeFSOption, -) (CinodeFS, error) { + options ...Option, +) (FS, error) { if be == nil { return nil, ErrInvalidBE } @@ -156,16 +158,16 @@ func (fs *cinodeFS) SetEntryFile( data io.Reader, opts ...EntrypointOption, ) (*Entrypoint, error) { - protoEntrypoint, err := protoEntrypointFromOptions(ctx, opts...) + ep, err := entrypointFromOptions(ctx, opts...) if err != nil { return nil, err } - if protoEntrypoint.MimeType == "" && len(path) > 0 { + if ep.ep.MimeType == "" && len(path) > 0 { // Try detecting mime type from filename extension - protoEntrypoint.MimeType = mime.TypeByExtension(filepath.Ext(path[len(path)-1])) + ep.ep.MimeType = mime.TypeByExtension(filepath.Ext(path[len(path)-1])) } - ep, err := fs.createFileEntrypoint(ctx, data, protoEntrypoint) + ep, err = fs.createFileEntrypoint(ctx, data, ep) if err != nil { return nil, err } @@ -183,7 +185,7 @@ func (fs *cinodeFS) CreateFileEntrypoint( data io.Reader, opts ...EntrypointOption, ) (*Entrypoint, error) { - ep, err := protoEntrypointFromOptions(ctx, opts...) + ep, err := entrypointFromOptions(ctx, opts...) if err != nil { return nil, err } @@ -194,11 +196,11 @@ func (fs *cinodeFS) CreateFileEntrypoint( func (fs *cinodeFS) createFileEntrypoint( ctx context.Context, data io.Reader, - protoEntrypoint *protobuf.Entrypoint, + ep *Entrypoint, ) (*Entrypoint, error) { var hw headWriter - if protoEntrypoint.MimeType == "" { + if ep.ep.MimeType == "" { // detect mimetype from the content hw = newHeadWriter(512) data = io.TeeReader(data, &hw) @@ -209,12 +211,11 @@ func (fs *cinodeFS) createFileEntrypoint( return nil, err } - if protoEntrypoint.MimeType == "" { - protoEntrypoint.MimeType = http.DetectContentType(hw.data) + if ep.ep.MimeType == "" { + ep.ep.MimeType = http.DetectContentType(hw.data) } - ep := entrypointFromBlobNameKeyAndProtoEntrypoint(bn, key, protoEntrypoint) - return ep, nil + return setEntrypointBlobNameAndKey(bn, key, ep), nil } func (fs *cinodeFS) SetEntry( @@ -230,7 +231,7 @@ func (fs *cinodeFS) SetEntry( if !isWriteable { return nil, 0, ErrMissingWriterInfo } - return &nodeUnloaded{ep: *ep}, dsDirty, nil + return &nodeUnloaded{ep: ep}, dsDirty, nil } return fs.traverseGraph( @@ -276,7 +277,7 @@ func (fs *cinodeFS) Flush(ctx context.Context) error { return err } - fs.rootEP = &nodeUnloaded{ep: *newRootEP} + fs.rootEP = &nodeUnloaded{ep: newRootEP} return nil } @@ -361,30 +362,30 @@ func (fs *cinodeFS) RootEntrypoint() (*Entrypoint, error) { return fs.rootEP.entrypoint() } -func (fs *cinodeFS) EntrypointWriterInfo(ctx context.Context, ep *Entrypoint) (WriterInfo, error) { +func (fs *cinodeFS) EntrypointWriterInfo(ctx context.Context, ep *Entrypoint) (*WriterInfo, error) { if !ep.IsLink() { - return WriterInfo{}, ErrNotALink + return nil, ErrNotALink } bn := ep.BlobName() key, err := fs.c.keyFromEntrypoint(ctx, ep) if err != nil { - return WriterInfo{}, err + return nil, err } authInfo, found := fs.c.writerInfos[bn.String()] if !found { - return WriterInfo{}, ErrMissingWriterInfo + return nil, ErrMissingWriterInfo } return writerInfoFromBlobNameKeyAndAuthInfo(bn, key, authInfo), nil } -func (fs *cinodeFS) RootWriterInfo(ctx context.Context) (WriterInfo, error) { +func (fs *cinodeFS) RootWriterInfo(ctx context.Context) (*WriterInfo, error) { rootEP, err := fs.RootEntrypoint() if err != nil { - return WriterInfo{}, err + return nil, err } return fs.EntrypointWriterInfo(ctx, rootEP) diff --git a/pkg/structure/graph/cinodefs_blackbox_test.go b/pkg/cinodefs/cinodefs_blackbox_test.go similarity index 89% rename from pkg/structure/graph/cinodefs_blackbox_test.go rename to pkg/cinodefs/cinodefs_blackbox_test.go index d3227d3..861a6ae 100644 --- a/pkg/structure/graph/cinodefs_blackbox_test.go +++ b/pkg/cinodefs/cinodefs_blackbox_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package graph_test +package cinodefs_test import ( "context" @@ -24,17 +24,17 @@ import ( "testing" "github.com/cinode/go/pkg/blenc" + "github.com/cinode/go/pkg/cinodefs" "github.com/cinode/go/pkg/datastore" - "github.com/cinode/go/pkg/structure/graph" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) func TestCinodeFSSingleFileScenario(t *testing.T) { ctx := context.Background() - fs, err := graph.NewCinodeFS(ctx, + fs, err := cinodefs.New(ctx, blenc.FromDatastore(datastore.InMemory()), - graph.NewRootDynamicLink(), + cinodefs.NewRootDynamicLink(), ) require.NoError(t, err) require.NotNil(t, fs) @@ -61,7 +61,7 @@ func TestCinodeFSSingleFileScenario(t *testing.T) { // Directories are modified, not yet flushed for i := range path1 { ep3, err := fs.FindEntry(ctx, path1[:i]) - require.ErrorIs(t, err, graph.ErrModifiedDirectory) + require.ErrorIs(t, err, cinodefs.ErrModifiedDirectory) require.Nil(t, ep3) } @@ -80,7 +80,7 @@ type CinodeFSMultiFileTestSuite struct { suite.Suite ds datastore.DS - fs graph.CinodeFS + fs cinodefs.FS contentMap []testFileEntry } @@ -92,9 +92,9 @@ func (c *CinodeFSMultiFileTestSuite) SetupTest() { ctx := context.Background() c.ds = datastore.InMemory() - fs, err := graph.NewCinodeFS(ctx, + fs, err := cinodefs.New(ctx, blenc.FromDatastore(c.ds), - graph.NewRootDynamicLink(), + cinodefs.NewRootDynamicLink(), ) require.NoError(c.T(), err) require.NotNil(c.T(), fs) @@ -127,7 +127,7 @@ func (c *CinodeFSMultiFileTestSuite) SetupTest() { c.checkContentMap(c.fs) } -func (c *CinodeFSMultiFileTestSuite) checkContentMap(fs graph.CinodeFS) { +func (c *CinodeFSMultiFileTestSuite) checkContentMap(fs cinodefs.FS) { ctx := context.Background() for _, file := range c.contentMap { ep, err := fs.FindEntry(ctx, file.path) @@ -150,10 +150,10 @@ func (c *CinodeFSMultiFileTestSuite) TestReopeningInReadOnlyMode() { rootEP, err := c.fs.RootEntrypoint() require.NoError(c.T(), err) - fs2, err := graph.NewCinodeFS( + fs2, err := cinodefs.New( ctx, blenc.FromDatastore(c.ds), - graph.RootEntrypoint(rootEP), + cinodefs.RootEntrypoint(rootEP), ) require.NoError(c.T(), err) require.NotNil(c.T(), fs2) @@ -173,10 +173,10 @@ func (c *CinodeFSMultiFileTestSuite) TestReopeningInReadOnlyMode() { require.NoError(c.T(), err) // reopen fs2 to avoid any caching issues - fs2, err = graph.NewCinodeFS( + fs2, err = cinodefs.New( ctx, blenc.FromDatastore(c.ds), - graph.RootEntrypoint(rootEP), + cinodefs.RootEntrypoint(rootEP), ) require.NoError(c.T(), err) @@ -186,7 +186,7 @@ func (c *CinodeFSMultiFileTestSuite) TestReopeningInReadOnlyMode() { // We should not be allowed to modify fs2 without writer info ep, err := fs2.SetEntryFile(ctx, c.contentMap[0].path, strings.NewReader("should fail")) - require.ErrorIs(c.T(), err, graph.ErrMissingWriterInfo) + require.ErrorIs(c.T(), err, cinodefs.ErrMissingWriterInfo) require.Nil(c.T(), ep) c.checkContentMap(c.fs) c.checkContentMap(fs2) @@ -199,10 +199,10 @@ func (c *CinodeFSMultiFileTestSuite) TestReopeningInReadWriteMode() { require.NoError(c.T(), err) require.NotNil(c.T(), rootWriterInfo) - fs3, err := graph.NewCinodeFS( + fs3, err := cinodefs.New( ctx, blenc.FromDatastore(c.ds), - graph.RootWriterInfo(rootWriterInfo), + cinodefs.RootWriterInfo(rootWriterInfo), ) require.NoError(c.T(), err) require.NotNil(c.T(), fs3) @@ -252,7 +252,7 @@ func (c *CinodeFSMultiFileTestSuite) TestRemovalOfADirectory() { c.checkContentMap(c.fs) err = c.fs.DeleteEntry(ctx, removedPath) - require.ErrorIs(c.T(), err, graph.ErrEntryNotFound) + require.ErrorIs(c.T(), err, cinodefs.ErrEntryNotFound) c.checkContentMap(c.fs) } @@ -262,7 +262,7 @@ func (c *CinodeFSMultiFileTestSuite) TestDeleteTreatFileAsDirectory() { path := append(c.contentMap[0].path, "sub-file") err := c.fs.DeleteEntry(ctx, path) - require.ErrorIs(c.T(), err, graph.ErrNotADirectory) + require.ErrorIs(c.T(), err, cinodefs.ErrNotADirectory) } func (c *CinodeFSMultiFileTestSuite) TestPreventSettingFileAsDirectory() { @@ -270,7 +270,7 @@ func (c *CinodeFSMultiFileTestSuite) TestPreventSettingFileAsDirectory() { path := append(c.contentMap[0].path, "sub-file") _, err := c.fs.SetEntryFile(ctx, path, strings.NewReader("should not happen")) - require.ErrorIs(c.T(), err, graph.ErrNotADirectory) + require.ErrorIs(c.T(), err, cinodefs.ErrNotADirectory) } func (c *CinodeFSMultiFileTestSuite) TestPreventSettingEmptyEntryName() { @@ -283,7 +283,7 @@ func (c *CinodeFSMultiFileTestSuite) TestPreventSettingEmptyEntryName() { } { c.T().Run(strings.Join(path, "::"), func(t *testing.T) { _, err := c.fs.SetEntryFile(ctx, path, strings.NewReader("should not succeed")) - require.ErrorIs(t, err, graph.ErrEmptyName) + require.ErrorIs(t, err, cinodefs.ErrEmptyName) }) } @@ -320,9 +320,9 @@ func (c *CinodeFSMultiFileTestSuite) TestRootEPDirectoryOnDirtyFS() { rootDir, err := c.fs.FindEntry(ctx, []string{}) require.NoError(c.T(), err) - fs2, err := graph.NewCinodeFS(ctx, + fs2, err := cinodefs.New(ctx, blenc.FromDatastore(c.ds), - graph.RootEntrypoint(rootDir), + cinodefs.RootEntrypoint(rootDir), ) require.NoError(c.T(), err) @@ -334,7 +334,7 @@ func (c *CinodeFSMultiFileTestSuite) TestRootEPDirectoryOnDirtyFS() { require.NoError(c.T(), err) ep2, err := fs2.RootEntrypoint() - require.ErrorIs(c.T(), err, graph.ErrModifiedDirectory) + require.ErrorIs(c.T(), err, cinodefs.ErrModifiedDirectory) require.Nil(c.T(), ep2) err = fs2.Flush(ctx) diff --git a/pkg/structure/graph/cinodefs_options.go b/pkg/cinodefs/cinodefs_options.go similarity index 81% rename from pkg/structure/graph/cinodefs_options.go rename to pkg/cinodefs/cinodefs_options.go index cec7c2a..8056afb 100644 --- a/pkg/structure/graph/cinodefs_options.go +++ b/pkg/cinodefs/cinodefs_options.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package graph +package cinodefs import ( "context" @@ -28,7 +28,7 @@ const ( DefaultMaxLinksRedirects = 10 ) -type CinodeFSOption interface { +type Option interface { apply(ctx context.Context, fs *cinodeFS) error } @@ -38,25 +38,25 @@ func (f optionFunc) apply(ctx context.Context, fs *cinodeFS) error { return f(ctx, fs) } -func MaxLinkRedirects(maxLinkRedirects int) CinodeFSOption { +func MaxLinkRedirects(maxLinkRedirects int) Option { return optionFunc(func(ctx context.Context, fs *cinodeFS) error { fs.maxLinkRedirects = maxLinkRedirects return nil }) } -func RootEntrypoint(ep *Entrypoint) CinodeFSOption { +func RootEntrypoint(ep *Entrypoint) Option { return optionFunc(func(ctx context.Context, fs *cinodeFS) error { - fs.rootEP = &nodeUnloaded{ep: *ep} + fs.rootEP = &nodeUnloaded{ep: ep} return nil }) } -func errOption(err error) CinodeFSOption { +func errOption(err error) Option { return optionFunc(func(ctx context.Context, fs *cinodeFS) error { return err }) } -func RootEntrypointString(eps string) CinodeFSOption { +func RootEntrypointString(eps string) Option { ep, err := EntrypointFromString(eps) if err != nil { return errOption(err) @@ -64,7 +64,7 @@ func RootEntrypointString(eps string) CinodeFSOption { return RootEntrypoint(ep) } -func RootWriterInfo(wi WriterInfo) CinodeFSOption { +func RootWriterInfo(wi *WriterInfo) Option { bn, err := common.BlobNameFromBytes(wi.wi.BlobName) if err != nil { return errOption(err) @@ -74,13 +74,13 @@ func RootWriterInfo(wi WriterInfo) CinodeFSOption { ep := EntrypointFromBlobNameAndKey(bn, key) return optionFunc(func(ctx context.Context, fs *cinodeFS) error { - fs.rootEP = &nodeUnloaded{ep: *ep} + fs.rootEP = &nodeUnloaded{ep: ep} fs.c.writerInfos[bn.String()] = wi.wi.AuthInfo return nil }) } -func RootWriterInfoString(wis string) CinodeFSOption { +func RootWriterInfoString(wis string) Option { wi, err := WriterInfoFromString(wis) if err != nil { return errOption(err) @@ -89,14 +89,14 @@ func RootWriterInfoString(wis string) CinodeFSOption { return RootWriterInfo(wi) } -func TimeFunc(f func() time.Time) CinodeFSOption { +func TimeFunc(f func() time.Time) Option { return optionFunc(func(ctx context.Context, fs *cinodeFS) error { fs.timeFunc = f return nil }) } -func RandSource(r io.Reader) CinodeFSOption { +func RandSource(r io.Reader) Option { return optionFunc(func(ctx context.Context, fs *cinodeFS) error { fs.randSource = r return nil @@ -105,7 +105,7 @@ func RandSource(r io.Reader) CinodeFSOption { // NewRootDynamicLink option can be used to create completely new, random // dynamic link as the root -func NewRootDynamicLink() CinodeFSOption { +func NewRootDynamicLink() Option { return optionFunc(func(ctx context.Context, fs *cinodeFS) error { newLinkEntrypoint, err := fs.GenerateNewDynamicLinkEntrypoint() if err != nil { @@ -117,7 +117,7 @@ func NewRootDynamicLink() CinodeFSOption { // creation and have to be flushed first to generate any // blobs fs.rootEP = &nodeLink{ - ep: *newLinkEntrypoint, + ep: newLinkEntrypoint, dState: dsSubDirty, target: &nodeDirectory{ entries: map[string]node{}, @@ -130,7 +130,7 @@ func NewRootDynamicLink() CinodeFSOption { // NewRootDynamicLink option can be used to create completely new, random // dynamic link as the root -func NewRootStaticDirectory() CinodeFSOption { +func NewRootStaticDirectory() Option { return optionFunc(func(ctx context.Context, fs *cinodeFS) error { fs.rootEP = &nodeDirectory{ entries: map[string]node{}, diff --git a/pkg/structure/graph/cinodefs_traverse.go b/pkg/cinodefs/cinodefs_traverse.go similarity index 98% rename from pkg/structure/graph/cinodefs_traverse.go rename to pkg/cinodefs/cinodefs_traverse.go index 5ed03e4..0f8fb24 100644 --- a/pkg/structure/graph/cinodefs_traverse.go +++ b/pkg/cinodefs/cinodefs_traverse.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package graph +package cinodefs import ( "context" diff --git a/pkg/structure/graph/context.go b/pkg/cinodefs/context.go similarity index 95% rename from pkg/structure/graph/context.go rename to pkg/cinodefs/context.go index 22240c5..4606502 100644 --- a/pkg/structure/graph/context.go +++ b/pkg/cinodefs/context.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package graph +package cinodefs import ( "bytes" @@ -24,8 +24,8 @@ import ( "io" "github.com/cinode/go/pkg/blenc" + "github.com/cinode/go/pkg/cinodefs/internal/protobuf" "github.com/cinode/go/pkg/common" - "github.com/cinode/go/pkg/structure/internal/protobuf" "google.golang.org/protobuf/proto" ) @@ -51,8 +51,7 @@ func (c *graphContext) keyFromEntrypoint( ctx context.Context, ep *Entrypoint, ) (common.BlobKey, error) { - if ep.ep == nil || - ep.ep.KeyInfo == nil || + if ep.ep.KeyInfo == nil || ep.ep.KeyInfo.Key == nil { return common.BlobKey{}, ErrMissingKeyInfo } @@ -127,7 +126,7 @@ func (c *graphContext) createProtobufMessage( return &Entrypoint{ bn: bn, - ep: &protobuf.Entrypoint{ + ep: protobuf.Entrypoint{ BlobName: bn.Bytes(), KeyInfo: &protobuf.KeyInfo{ Key: key.Bytes(), diff --git a/pkg/structure/graph/entrypoint.go b/pkg/cinodefs/entrypoint.go similarity index 76% rename from pkg/structure/graph/entrypoint.go rename to pkg/cinodefs/entrypoint.go index 685a8f5..4a0314a 100644 --- a/pkg/structure/graph/entrypoint.go +++ b/pkg/cinodefs/entrypoint.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package graph +package cinodefs import ( "errors" @@ -22,8 +22,8 @@ import ( "time" "github.com/cinode/go/pkg/blobtypes" + "github.com/cinode/go/pkg/cinodefs/internal/protobuf" "github.com/cinode/go/pkg/common" - "github.com/cinode/go/pkg/structure/internal/protobuf" "github.com/cinode/go/pkg/utilities/golang" "github.com/jbenet/go-base58" "google.golang.org/protobuf/proto" @@ -40,7 +40,7 @@ var ( ) type Entrypoint struct { - ep *protobuf.Entrypoint + ep protobuf.Entrypoint bn common.BlobName } @@ -58,14 +58,19 @@ func EntrypointFromString(s string) (*Entrypoint, error) { } func EntrypointFromBytes(b []byte) (*Entrypoint, error) { - data := protobuf.Entrypoint{} + ep := &Entrypoint{} - err := proto.Unmarshal(b, &data) + err := proto.Unmarshal(b, &ep.ep) if err != nil { return nil, fmt.Errorf("%w: %s", ErrInvalidEntrypointDataParse, err) } - return entrypointFromProtobuf(&data) + err = expandEntrypointProto(ep) + if err != nil { + return nil, err + } + + return ep, nil } func entrypointFromProtobuf(data *protobuf.Entrypoint) (*Entrypoint, error) { @@ -73,35 +78,40 @@ func entrypointFromProtobuf(data *protobuf.Entrypoint) (*Entrypoint, error) { return nil, ErrInvalidEntrypointDataNil } - ret := Entrypoint{ep: data} + ep := &Entrypoint{} + proto.Merge(&ep.ep, data) + err := expandEntrypointProto(ep) + if err != nil { + return nil, err + } + return ep, nil +} +func expandEntrypointProto(ep *Entrypoint) error { // Extract blob name from entrypoint - bn, err := common.BlobNameFromBytes(data.BlobName) + bn, err := common.BlobNameFromBytes(ep.ep.BlobName) if err != nil { - return nil, fmt.Errorf("%w: %s", ErrInvalidEntrypointData, err) + return fmt.Errorf("%w: %s", ErrInvalidEntrypointData, err) } - ret.bn = bn + ep.bn = bn // Links must not have mimetype set - if ret.IsLink() && data.MimeType != "" { - return nil, ErrInvalidEntrypointDataLinkMimetype + if ep.IsLink() && ep.ep.MimeType != "" { + return ErrInvalidEntrypointDataLinkMimetype } - return &ret, nil + return nil } func EntrypointFromBlobNameAndKey(bn common.BlobName, key common.BlobKey) *Entrypoint { - return entrypointFromBlobNameKeyAndProtoEntrypoint(bn, key, &protobuf.Entrypoint{}) + return setEntrypointBlobNameAndKey(bn, key, &Entrypoint{}) } -func entrypointFromBlobNameKeyAndProtoEntrypoint(bn common.BlobName, key common.BlobKey, protoEp *protobuf.Entrypoint) *Entrypoint { - protoEp.BlobName = bn.Bytes() - protoEp.KeyInfo = &protobuf.KeyInfo{Key: key.Bytes()} - - return &Entrypoint{ - ep: protoEp, - bn: bn, - } +func setEntrypointBlobNameAndKey(bn common.BlobName, key common.BlobKey, ep *Entrypoint) *Entrypoint { + ep.bn = bn + ep.ep.BlobName = bn.Bytes() + ep.ep.KeyInfo = &protobuf.KeyInfo{Key: key.Bytes()} + return ep } func (e *Entrypoint) String() string { @@ -109,7 +119,7 @@ func (e *Entrypoint) String() string { } func (e *Entrypoint) Bytes() []byte { - return golang.Must(proto.Marshal(e.ep)) + return golang.Must(proto.Marshal(&e.ep)) } func (e *Entrypoint) BlobName() common.BlobName { diff --git a/pkg/structure/graph/entrypoint_options.go b/pkg/cinodefs/entrypoint_options.go similarity index 50% rename from pkg/structure/graph/entrypoint_options.go rename to pkg/cinodefs/entrypoint_options.go index 32cac4e..57505c1 100644 --- a/pkg/structure/graph/entrypoint_options.go +++ b/pkg/cinodefs/entrypoint_options.go @@ -14,54 +14,48 @@ See the License for the specific language governing permissions and limitations under the License. */ -package graph +package cinodefs import ( "context" "time" - - "github.com/cinode/go/pkg/structure/internal/protobuf" ) type EntrypointOption interface { - apply(ctx context.Context, opts *entrypointOptions) error + apply(ctx context.Context, ep *Entrypoint) error } -type entrypointOptionBasicFunc func(opts *entrypointOptions) +type entrypointOptionBasicFunc func(ep *Entrypoint) -func (ep entrypointOptionBasicFunc) apply(ctx context.Context, opts *entrypointOptions) error { - ep(opts) +func (f entrypointOptionBasicFunc) apply(ctx context.Context, ep *Entrypoint) error { + f(ep) return nil } -type entrypointOptions struct { - ep *protobuf.Entrypoint -} - func SetMimeType(mimeType string) EntrypointOption { - return entrypointOptionBasicFunc(func(ep *entrypointOptions) { + return entrypointOptionBasicFunc(func(ep *Entrypoint) { ep.ep.MimeType = mimeType }) } func SetNotValidBefore(t time.Time) EntrypointOption { - return entrypointOptionBasicFunc(func(opts *entrypointOptions) { - opts.ep.NotValidBeforeUnixMicro = t.UnixMicro() + return entrypointOptionBasicFunc(func(ep *Entrypoint) { + ep.ep.NotValidBeforeUnixMicro = t.UnixMicro() }) } func SetNotValidAfter(t time.Time) EntrypointOption { - return entrypointOptionBasicFunc(func(opts *entrypointOptions) { - opts.ep.NotValidAfterUnixMicro = t.UnixMicro() + return entrypointOptionBasicFunc(func(ep *Entrypoint) { + ep.ep.NotValidAfterUnixMicro = t.UnixMicro() }) } -func protoEntrypointFromOptions(ctx context.Context, opts ...EntrypointOption) (*protobuf.Entrypoint, error) { - scratchpad := entrypointOptions{ep: &protobuf.Entrypoint{}} +func entrypointFromOptions(ctx context.Context, opts ...EntrypointOption) (*Entrypoint, error) { + ep := &Entrypoint{} for _, o := range opts { - if err := o.apply(ctx, &scratchpad); err != nil { + if err := o.apply(ctx, ep); err != nil { return nil, err } } - return scratchpad.ep, nil + return ep, nil } diff --git a/pkg/structure/graph/headwriter.go b/pkg/cinodefs/headwriter.go similarity index 98% rename from pkg/structure/graph/headwriter.go rename to pkg/cinodefs/headwriter.go index bbe8ab1..13f0973 100644 --- a/pkg/structure/graph/headwriter.go +++ b/pkg/cinodefs/headwriter.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package graph +package cinodefs type headWriter struct { limit int diff --git a/pkg/structure/graphutils/http.go b/pkg/cinodefs/httphandler/http.go similarity index 89% rename from pkg/structure/graphutils/http.go rename to pkg/cinodefs/httphandler/http.go index f10ba91..c445d75 100644 --- a/pkg/structure/graphutils/http.go +++ b/pkg/cinodefs/httphandler/http.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package graphutils +package httphandler import ( "errors" @@ -24,17 +24,17 @@ import ( "net/url" "strings" - "github.com/cinode/go/pkg/structure/graph" + "github.com/cinode/go/pkg/cinodefs" "golang.org/x/exp/slog" ) -type HTTPHandler struct { - FS graph.CinodeFS +type Handler struct { + FS cinodefs.FS IndexFile string Log *slog.Logger } -func (h *HTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { log := h.Log.With( slog.String("RemoteAddr", r.RemoteAddr), slog.String("URL", r.URL.String()), @@ -71,8 +71,8 @@ func (h *HTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { fileEP, err := h.FS.FindEntry(r.Context(), pathList) switch { - case errors.Is(err, graph.ErrEntryNotFound), - errors.Is(err, graph.ErrNotADirectory): + case errors.Is(err, cinodefs.ErrEntryNotFound), + errors.Is(err, cinodefs.ErrNotADirectory): log.Warn("Not found") http.NotFound(w, r) return diff --git a/pkg/structure/internal/protobuf/protobuf.go b/pkg/cinodefs/internal/protobuf/protobuf.go similarity index 100% rename from pkg/structure/internal/protobuf/protobuf.go rename to pkg/cinodefs/internal/protobuf/protobuf.go diff --git a/pkg/structure/internal/protobuf/protobuf.pb.go b/pkg/cinodefs/internal/protobuf/protobuf.pb.go similarity index 100% rename from pkg/structure/internal/protobuf/protobuf.pb.go rename to pkg/cinodefs/internal/protobuf/protobuf.pb.go diff --git a/pkg/structure/internal/protobuf/protobuf.proto b/pkg/cinodefs/internal/protobuf/protobuf.proto similarity index 100% rename from pkg/structure/internal/protobuf/protobuf.proto rename to pkg/cinodefs/internal/protobuf/protobuf.proto diff --git a/pkg/structure/graph/node.go b/pkg/cinodefs/node.go similarity index 99% rename from pkg/structure/graph/node.go rename to pkg/cinodefs/node.go index d07a5ca..6bbe9b2 100644 --- a/pkg/structure/graph/node.go +++ b/pkg/cinodefs/node.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package graph +package cinodefs // // cached entries: diff --git a/pkg/structure/graph/node_directory.go b/pkg/cinodefs/node_directory.go similarity index 98% rename from pkg/structure/graph/node_directory.go rename to pkg/cinodefs/node_directory.go index b66bc74..8ba8c1e 100644 --- a/pkg/structure/graph/node_directory.go +++ b/pkg/cinodefs/node_directory.go @@ -14,14 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package graph +package cinodefs import ( "context" "sort" "github.com/cinode/go/pkg/blobtypes" - "github.com/cinode/go/pkg/structure/internal/protobuf" + "github.com/cinode/go/pkg/cinodefs/internal/protobuf" "github.com/cinode/go/pkg/utilities/golang" ) @@ -79,7 +79,7 @@ func (d *nodeDirectory) flush(ctx context.Context, gc *graphContext) (node, *Ent flushedEntries[name] = target dir.Entries = append(dir.Entries, &protobuf.Directory_Entry{ Name: name, - Ep: targetEP.ep, + Ep: &targetEP.ep, }) } diff --git a/pkg/structure/graph/node_file.go b/pkg/cinodefs/node_file.go similarity index 94% rename from pkg/structure/graph/node_file.go rename to pkg/cinodefs/node_file.go index 6f98ea6..77347e5 100644 --- a/pkg/structure/graph/node_file.go +++ b/pkg/cinodefs/node_file.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package graph +package cinodefs import ( "context" @@ -22,7 +22,7 @@ import ( // Entry is a file with its entrypoint type nodeFile struct { - ep Entrypoint + ep *Entrypoint } func (c *nodeFile) dirty() dirtyState { @@ -30,7 +30,7 @@ func (c *nodeFile) dirty() dirtyState { } func (c *nodeFile) flush(ctx context.Context, gc *graphContext) (node, *Entrypoint, error) { - return c, &c.ep, nil + return c, c.ep, nil } func (c *nodeFile) traverse( @@ -56,5 +56,5 @@ func (c *nodeFile) traverse( } func (c *nodeFile) entrypoint() (*Entrypoint, error) { - return &c.ep, nil + return c.ep, nil } diff --git a/pkg/structure/graph/node_link.go b/pkg/cinodefs/node_link.go similarity index 91% rename from pkg/structure/graph/node_link.go rename to pkg/cinodefs/node_link.go index b19cddf..9a1e81e 100644 --- a/pkg/structure/graph/node_link.go +++ b/pkg/cinodefs/node_link.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package graph +package cinodefs import ( "context" @@ -24,8 +24,8 @@ import ( // Entry is a link loaded into memory type nodeLink struct { - ep Entrypoint // entrypoint of the link itself - target node // target for the link + ep *Entrypoint // entrypoint of the link itself + target node // target for the link dState dirtyState } @@ -36,7 +36,7 @@ func (c *nodeLink) dirty() dirtyState { func (c *nodeLink) flush(ctx context.Context, gc *graphContext) (node, *Entrypoint, error) { if c.dState == dsClean { // all clear - return c, &c.ep, nil + return c, c.ep, nil } golang.Assert(c.dState == dsSubDirty, "link can be clean or sub-dirty") @@ -45,7 +45,7 @@ func (c *nodeLink) flush(ctx context.Context, gc *graphContext) (node, *Entrypoi return nil, nil, err } - err = gc.updateProtobufMessage(ctx, &c.ep, targetEP.ep) + err = gc.updateProtobufMessage(ctx, c.ep, &targetEP.ep) if err != nil { return nil, nil, err } @@ -56,7 +56,7 @@ func (c *nodeLink) flush(ctx context.Context, gc *graphContext) (node, *Entrypoi dState: dsClean, } - return ret, &ret.ep, nil + return ret, ret.ep, nil } func (c *nodeLink) traverse( @@ -123,5 +123,5 @@ func (c *nodeLink) traverse( } func (c *nodeLink) entrypoint() (*Entrypoint, error) { - return &c.ep, nil + return c.ep, nil } diff --git a/pkg/structure/graph/node_unloaded.go b/pkg/cinodefs/node_unloaded.go similarity index 85% rename from pkg/structure/graph/node_unloaded.go rename to pkg/cinodefs/node_unloaded.go index 94047a1..0d5fe3b 100644 --- a/pkg/structure/graph/node_unloaded.go +++ b/pkg/cinodefs/node_unloaded.go @@ -14,17 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */ -package graph +package cinodefs import ( "context" "fmt" - "github.com/cinode/go/pkg/structure/internal/protobuf" + "github.com/cinode/go/pkg/cinodefs/internal/protobuf" ) type nodeUnloaded struct { - ep Entrypoint + ep *Entrypoint } func (c *nodeUnloaded) dirty() dirtyState { @@ -32,7 +32,7 @@ func (c *nodeUnloaded) dirty() dirtyState { } func (c *nodeUnloaded) flush(ctx context.Context, gc *graphContext) (node, *Entrypoint, error) { - return c, &c.ep, nil + return c, c.ep, nil } func (c *nodeUnloaded) traverse( @@ -80,27 +80,27 @@ func (c *nodeUnloaded) load(ctx context.Context, gc *graphContext) (node, error) } func (c *nodeUnloaded) loadEntrypointLink(ctx context.Context, gc *graphContext) (node, error) { - msg := &protobuf.Entrypoint{} - err := gc.readProtobufMessage(ctx, &c.ep, msg) + targetEP := &Entrypoint{} + err := gc.readProtobufMessage(ctx, c.ep, &targetEP.ep) if err != nil { return nil, err } - targetEP, err := entrypointFromProtobuf(msg) + err = expandEntrypointProto(targetEP) if err != nil { return nil, err } return &nodeLink{ ep: c.ep, - target: &nodeUnloaded{ep: *targetEP}, + target: &nodeUnloaded{ep: targetEP}, dState: dsClean, }, nil } func (c *nodeUnloaded) loadEntrypointDir(ctx context.Context, gc *graphContext) (node, error) { msg := &protobuf.Directory{} - err := gc.readProtobufMessage(ctx, &c.ep, msg) + err := gc.readProtobufMessage(ctx, c.ep, msg) if err != nil { return nil, err } @@ -120,16 +120,16 @@ func (c *nodeUnloaded) loadEntrypointDir(ctx context.Context, gc *graphContext) return nil, err } - dir[entry.Name] = &nodeUnloaded{ep: *ep} + dir[entry.Name] = &nodeUnloaded{ep: ep} } return &nodeDirectory{ - stored: &c.ep, + stored: c.ep, entries: dir, dState: dsClean, }, nil } func (c *nodeUnloaded) entrypoint() (*Entrypoint, error) { - return &c.ep, nil + return c.ep, nil } diff --git a/pkg/structure/graphutils/directory.go b/pkg/cinodefs/uploader/directory.go similarity index 90% rename from pkg/structure/graphutils/directory.go rename to pkg/cinodefs/uploader/directory.go index 9a95081..1b49dca 100644 --- a/pkg/structure/graphutils/directory.go +++ b/pkg/cinodefs/uploader/directory.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package graphutils +package uploader import ( "bytes" @@ -28,7 +28,7 @@ import ( _ "embed" "github.com/cinode/go/pkg/blenc" - "github.com/cinode/go/pkg/structure/graph" + "github.com/cinode/go/pkg/cinodefs" "github.com/cinode/go/pkg/utilities/golang" "golang.org/x/exp/slog" ) @@ -46,8 +46,8 @@ var ( func UploadStaticDirectory( ctx context.Context, fsys fs.FS, - cfs graph.CinodeFS, - opts ...UploadStaticDirectoryOption, + cfs cinodefs.FS, + opts ...Option, ) error { c := dirCompiler{ ctx: ctx, @@ -74,17 +74,17 @@ func UploadStaticDirectory( return nil } -type UploadStaticDirectoryOption func(d *dirCompiler) error +type Option func(d *dirCompiler) error -func BasePath(path []string) UploadStaticDirectoryOption { - return UploadStaticDirectoryOption(func(d *dirCompiler) error { +func BasePath(path []string) Option { + return Option(func(d *dirCompiler) error { d.basePath = path return nil }) } -func CreateIndexFile(indexFile string) UploadStaticDirectoryOption { - return UploadStaticDirectoryOption(func(d *dirCompiler) error { +func CreateIndexFile(indexFile string) Option { + return Option(func(d *dirCompiler) error { d.createIndexFile = true d.indexFileName = indexFile return nil @@ -94,7 +94,7 @@ func CreateIndexFile(indexFile string) UploadStaticDirectoryOption { type dirCompiler struct { ctx context.Context fsys fs.FS - cfs graph.CinodeFS + cfs cinodefs.FS log *slog.Logger basePath []string createIndexFile bool @@ -131,7 +131,7 @@ func (d *dirCompiler) compilePath( } return &dirEntry{ Name: name, - MimeType: graph.CinodeDirMimeType, + MimeType: cinodefs.CinodeDirMimeType, IsDir: true, Size: int64(size), }, nil diff --git a/pkg/structure/graphutils/templates/dir.html b/pkg/cinodefs/uploader/templates/dir.html similarity index 100% rename from pkg/structure/graphutils/templates/dir.html rename to pkg/cinodefs/uploader/templates/dir.html diff --git a/pkg/structure/graph/writerinfo.go b/pkg/cinodefs/writerinfo.go similarity index 62% rename from pkg/structure/graph/writerinfo.go rename to pkg/cinodefs/writerinfo.go index 7ccaada..f69ae71 100644 --- a/pkg/structure/graph/writerinfo.go +++ b/pkg/cinodefs/writerinfo.go @@ -14,14 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package graph +package cinodefs import ( "errors" "fmt" + "github.com/cinode/go/pkg/cinodefs/internal/protobuf" "github.com/cinode/go/pkg/common" - "github.com/cinode/go/pkg/structure/internal/protobuf" "github.com/cinode/go/pkg/utilities/golang" "github.com/jbenet/go-base58" "google.golang.org/protobuf/proto" @@ -33,48 +33,44 @@ var ( ) type WriterInfo struct { - wi *protobuf.WriterInfo + wi protobuf.WriterInfo } func (wi *WriterInfo) Bytes() []byte { - return golang.Must(proto.Marshal(wi.wi)) + return golang.Must(proto.Marshal(&wi.wi)) } func (wi *WriterInfo) String() string { return base58.Encode(wi.Bytes()) } -func WriterInfoFromString(s string) (WriterInfo, error) { +func WriterInfoFromString(s string) (*WriterInfo, error) { if len(s) == 0 { - return WriterInfo{}, fmt.Errorf("%w: empty string", ErrInvalidWriterInfoData) + return nil, fmt.Errorf("%w: empty string", ErrInvalidWriterInfoData) } b := base58.Decode(s) if len(b) == 0 { - return WriterInfo{}, fmt.Errorf("%w: not a base58 string", ErrInvalidWriterInfoData) + return nil, fmt.Errorf("%w: not a base58 string", ErrInvalidWriterInfoData) } return WriterInfoFromBytes(b) } -func WriterInfoFromBytes(b []byte) (WriterInfo, error) { - data := protobuf.WriterInfo{} +func WriterInfoFromBytes(b []byte) (*WriterInfo, error) { + wi := WriterInfo{} - err := proto.Unmarshal(b, &data) + err := proto.Unmarshal(b, &wi.wi) if err != nil { - return WriterInfo{}, fmt.Errorf("%w: %s", ErrInvalidWriterInfoDataParse, err) + return nil, fmt.Errorf("%w: %s", ErrInvalidWriterInfoDataParse, err) } - return writerInfoFromProtobuf(&data) + return &wi, nil } -func writerInfoFromProtobuf(data *protobuf.WriterInfo) (WriterInfo, error) { - return WriterInfo{wi: data}, nil -} - -func writerInfoFromBlobNameKeyAndAuthInfo(bn common.BlobName, key common.BlobKey, ai []byte) WriterInfo { - return WriterInfo{ - wi: &protobuf.WriterInfo{ +func writerInfoFromBlobNameKeyAndAuthInfo(bn common.BlobName, key common.BlobKey, ai []byte) *WriterInfo { + return &WriterInfo{ + wi: protobuf.WriterInfo{ BlobName: bn.Bytes(), Key: key.Bytes(), AuthInfo: ai, diff --git a/pkg/cmd/cinode_web_proxy/integration_test.go b/pkg/cmd/cinode_web_proxy/integration_test.go index 34acc0f..a76bdde 100644 --- a/pkg/cmd/cinode_web_proxy/integration_test.go +++ b/pkg/cmd/cinode_web_proxy/integration_test.go @@ -29,17 +29,17 @@ import ( "time" "github.com/cinode/go/pkg/blenc" + "github.com/cinode/go/pkg/cinodefs" + "github.com/cinode/go/pkg/cinodefs/uploader" "github.com/cinode/go/pkg/cmd/cinode_web_proxy" "github.com/cinode/go/pkg/datastore" - "github.com/cinode/go/pkg/structure" - "github.com/jbenet/go-base58" "github.com/stretchr/testify/require" - "golang.org/x/exp/slog" ) func TestIntegration(t *testing.T) { - // Prepare test filesystem + os.Clearenv() + // Prepare test filesystem testFS := fstest.MapFS{ "index.html": &fstest.MapFile{ Data: []byte("Hello world!"), @@ -64,18 +64,27 @@ func TestIntegration(t *testing.T) { ds, err := datastore.InRawFileSystem(dir) require.NoError(t, err) - ep, err := structure.UploadStaticDirectory( + cfs, err := cinodefs.New( context.Background(), - slog.Default(), - testFS, blenc.FromDatastore(ds), + cinodefs.NewRootStaticDirectory(), + ) + require.NoError(t, err) + + err = uploader.UploadStaticDirectory( + context.Background(), + testFS, + cfs, ) require.NoError(t, err) - epBytes, err := ep.ToBytes() + err = cfs.Flush(context.Background()) + require.NoError(t, err) + + ep, err := cfs.RootEntrypoint() require.NoError(t, err) - t.Setenv("CINODE_ENTRYPOINT", base58.Encode(epBytes)) + t.Setenv("CINODE_ENTRYPOINT", ep.String()) runAndValidateCinodeProxy := func() { ctx, cancel := context.WithCancel(context.Background()) diff --git a/pkg/cmd/cinode_web_proxy/root.go b/pkg/cmd/cinode_web_proxy/root.go index 8c587ec..274d522 100644 --- a/pkg/cmd/cinode_web_proxy/root.go +++ b/pkg/cmd/cinode_web_proxy/root.go @@ -29,9 +29,9 @@ import ( "time" "github.com/cinode/go/pkg/blenc" + "github.com/cinode/go/pkg/cinodefs" + "github.com/cinode/go/pkg/cinodefs/httphandler" "github.com/cinode/go/pkg/datastore" - "github.com/cinode/go/pkg/structure/graph" - "github.com/cinode/go/pkg/structure/graphutils" "github.com/cinode/go/pkg/utilities/httpserver" "golang.org/x/exp/slog" ) @@ -59,7 +59,7 @@ func executeWithConfig(ctx context.Context, cfg *config) error { additionalDSs = append(additionalDSs, ds) } - entrypoint, err := graph.EntrypointFromString(cfg.entrypoint) + entrypoint, err := cinodefs.EntrypointFromString(cfg.entrypoint) if err != nil { return fmt.Errorf("could not parse entrypoint data: %w", err) } @@ -95,9 +95,9 @@ func setupCinodeProxy( ctx context.Context, mainDS datastore.DS, additionalDSs []datastore.DS, - entrypoint *graph.Entrypoint, + entrypoint *cinodefs.Entrypoint, ) (http.Handler, error) { - fs, err := graph.NewCinodeFS( + fs, err := cinodefs.New( ctx, blenc.FromDatastore( datastore.NewMultiSource( @@ -106,14 +106,14 @@ func setupCinodeProxy( additionalDSs..., ), ), - graph.RootEntrypoint(entrypoint), - graph.MaxLinkRedirects(10), + cinodefs.RootEntrypoint(entrypoint), + cinodefs.MaxLinkRedirects(10), ) if err != nil { return nil, err } - return &graphutils.HTTPHandler{ + return &httphandler.Handler{ FS: fs, IndexFile: "index.html", Log: slog.Default(), diff --git a/pkg/cmd/cinode_web_proxy/root_test.go b/pkg/cmd/cinode_web_proxy/root_test.go index ef1e07f..908affa 100644 --- a/pkg/cmd/cinode_web_proxy/root_test.go +++ b/pkg/cmd/cinode_web_proxy/root_test.go @@ -30,11 +30,11 @@ import ( "github.com/cinode/go/pkg/blenc" "github.com/cinode/go/pkg/blobtypes" + "github.com/cinode/go/pkg/cinodefs" + "github.com/cinode/go/pkg/cinodefs/uploader" "github.com/cinode/go/pkg/common" "github.com/cinode/go/pkg/datastore" "github.com/cinode/go/pkg/internal/utilities/cipherfactory" - "github.com/cinode/go/pkg/structure/graph" - "github.com/cinode/go/pkg/structure/graphutils" "github.com/cinode/go/testvectors/testblobs" "github.com/jbenet/go-base58" "github.com/stretchr/testify/require" @@ -118,7 +118,7 @@ func TestWebProxyHandlerInvalidEntrypoint(t *testing.T) { context.Background(), datastore.InMemory(), []datastore.DS{}, - graph.EntrypointFromBlobNameAndKey(n, key), + cinodefs.EntrypointFromBlobNameAndKey(n, key), ) require.NoError(t, err) @@ -147,7 +147,7 @@ func TestWebProxyHandlerSimplePage(t *testing.T) { ds := datastore.InMemory() be := blenc.FromDatastore(ds) - ep := func() *graph.Entrypoint { + ep := func() *cinodefs.Entrypoint { dir := t.TempDir() for name, content := range map[string]string{ @@ -161,10 +161,10 @@ func TestWebProxyHandlerSimplePage(t *testing.T) { require.NoError(t, err) } - fs, err := graph.NewCinodeFS(context.Background(), be, graph.NewRootDynamicLink()) + fs, err := cinodefs.New(context.Background(), be, cinodefs.NewRootDynamicLink()) require.NoError(t, err) - err = graphutils.UploadStaticDirectory( + err = uploader.UploadStaticDirectory( context.Background(), os.DirFS(dir), fs, diff --git a/pkg/cmd/static_datastore/compile.go b/pkg/cmd/static_datastore/compile.go index 2b0ce6a..2de246b 100644 --- a/pkg/cmd/static_datastore/compile.go +++ b/pkg/cmd/static_datastore/compile.go @@ -26,9 +26,9 @@ import ( "strings" "github.com/cinode/go/pkg/blenc" + "github.com/cinode/go/pkg/cinodefs" + "github.com/cinode/go/pkg/cinodefs/uploader" "github.com/cinode/go/pkg/datastore" - "github.com/cinode/go/pkg/structure/graph" - "github.com/cinode/go/pkg/structure/graphutils" "github.com/spf13/cobra" ) @@ -77,11 +77,11 @@ func compileCmd() *cobra.Command { rootWriterInfoStr = string(data) } if len(rootWriterInfoStr) > 0 { - wi, err := graph.WriterInfoFromString(rootWriterInfoStr) + wi, err := cinodefs.WriterInfoFromString(rootWriterInfoStr) if err != nil { fatalResult("Couldn't parse writer info: %v", err) } - o.writerInfo = &wi + o.writerInfo = wi } if useRawFilesystem { @@ -160,7 +160,7 @@ type compileFSOptions struct { srcDir string dstLocation string static bool - writerInfo *graph.WriterInfo + writerInfo *cinodefs.WriterInfo generateIndexFiles bool indexFile string append bool @@ -170,8 +170,8 @@ func compileFS( ctx context.Context, o compileFSOptions, ) ( - *graph.Entrypoint, - *graph.WriterInfo, + *cinodefs.Entrypoint, + *cinodefs.WriterInfo, error, ) { ds, err := datastore.FromLocation(o.dstLocation) @@ -179,16 +179,16 @@ func compileFS( return nil, nil, fmt.Errorf("could not open datastore: %w", err) } - opts := []graph.CinodeFSOption{} + opts := []cinodefs.Option{} if o.static { - opts = append(opts, graph.NewRootStaticDirectory()) + opts = append(opts, cinodefs.NewRootStaticDirectory()) } else if o.writerInfo == nil { - opts = append(opts, graph.NewRootDynamicLink()) + opts = append(opts, cinodefs.NewRootDynamicLink()) } else { - opts = append(opts, graph.RootWriterInfo(*o.writerInfo)) + opts = append(opts, cinodefs.RootWriterInfo(o.writerInfo)) } - fs, err := graph.NewCinodeFS( + fs, err := cinodefs.New( ctx, blenc.FromDatastore(ds), opts..., @@ -204,12 +204,12 @@ func compileFS( } } - var genOpts []graphutils.UploadStaticDirectoryOption + var genOpts []uploader.Option if o.generateIndexFiles { - genOpts = append(genOpts, graphutils.CreateIndexFile(o.indexFile)) + genOpts = append(genOpts, uploader.CreateIndexFile(o.indexFile)) } - err = graphutils.UploadStaticDirectory( + err = uploader.UploadStaticDirectory( ctx, os.DirFS(o.srcDir), fs, @@ -225,12 +225,12 @@ func compileFS( } wi, err := fs.RootWriterInfo(ctx) - if errors.Is(err, graph.ErrNotALink) { + if errors.Is(err, cinodefs.ErrNotALink) { return ep, nil, nil } if err != nil { return nil, nil, fmt.Errorf("couldn't get root writer info from cinodefs instance: %w", err) } - return ep, &wi, nil + return ep, wi, nil } diff --git a/pkg/cmd/static_datastore/static_datastore_test.go b/pkg/cmd/static_datastore/static_datastore_test.go index 22c053c..00a710c 100644 --- a/pkg/cmd/static_datastore/static_datastore_test.go +++ b/pkg/cmd/static_datastore/static_datastore_test.go @@ -27,9 +27,9 @@ import ( "testing" "github.com/cinode/go/pkg/blenc" + "github.com/cinode/go/pkg/cinodefs" + "github.com/cinode/go/pkg/cinodefs/httphandler" "github.com/cinode/go/pkg/datastore" - "github.com/cinode/go/pkg/structure/graph" - "github.com/cinode/go/pkg/structure/graphutils" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "golang.org/x/exp/slog" @@ -102,10 +102,10 @@ func TestCompileAndReadTestSuite(t *testing.T) { func (s *CompileAndReadTestSuite) uploadDatasetToDatastore( dataset []datasetFile, datastoreDir string, - wi *graph.WriterInfo, -) (*graph.WriterInfo, *graph.Entrypoint) { + wi *cinodefs.WriterInfo, +) (*cinodefs.WriterInfo, *cinodefs.Entrypoint) { - var ep *graph.Entrypoint + var ep *cinodefs.Entrypoint s.T().Run("prepare dataset", func(t *testing.T) { dir := t.TempDir() @@ -133,21 +133,21 @@ func (s *CompileAndReadTestSuite) uploadDatasetToDatastore( func (s *CompileAndReadTestSuite) validateDataset( dataset []datasetFile, - ep *graph.Entrypoint, + ep *cinodefs.Entrypoint, datastoreDir string, ) { ds, err := datastore.InFileSystem(datastoreDir) s.Require().NoError(err) - fs, err := graph.NewCinodeFS( + fs, err := cinodefs.New( context.Background(), blenc.FromDatastore(ds), - graph.RootEntrypoint(ep), - graph.MaxLinkRedirects(10), + cinodefs.RootEntrypoint(ep), + cinodefs.MaxLinkRedirects(10), ) s.Require().NoError(err) - testServer := httptest.NewServer(&graphutils.HTTPHandler{ + testServer := httptest.NewServer(&httphandler.Handler{ FS: fs, IndexFile: "index.html", Log: slog.Default(), diff --git a/pkg/structure/cinodefs.go b/pkg/structure/cinodefs.go deleted file mode 100644 index 6e3ba95..0000000 --- a/pkg/structure/cinodefs.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Copyright © 2023 Bartłomiej Święcki (byo) - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package structure - -import ( - "context" - "io" - "strings" - "time" - - "github.com/cinode/go/pkg/blenc" - "github.com/cinode/go/pkg/structure/internal/protobuf" - "google.golang.org/protobuf/proto" -) - -type CinodeFS struct { - BE blenc.BE - RootEntrypoint *protobuf.Entrypoint - MaxLinkRedirects int - CurrentTimeF func() time.Time -} - -func (d *CinodeFS) OpenContent(ctx context.Context, ep *protobuf.Entrypoint) (io.ReadCloser, error) { - bn, key, err := ep.ValidateAndParse(time.Now()) - if err != nil { - return nil, err - } - - return d.BE.Open(ctx, bn, key) -} - -func (d *CinodeFS) FindEntrypoint(ctx context.Context, path string) (*protobuf.Entrypoint, error) { - return d.findEntrypointInDir(ctx, d.RootEntrypoint, path, d.currentTime()) -} - -func (d *CinodeFS) findEntrypointInDir( - ctx context.Context, - ep *protobuf.Entrypoint, - remainingPath string, - currentTime time.Time, -) ( - *protobuf.Entrypoint, - error, -) { - ep, err := DereferenceLink(ctx, d.BE, ep, d.MaxLinkRedirects, currentTime) - if err != nil { - return nil, err - } - - if ep.MimeType != CinodeDirMimeType { - return nil, ErrNotADirectory - } - - rc, err := d.OpenContent(ctx, ep) - if err != nil { - return nil, err - } - defer rc.Close() - - data, err := io.ReadAll(rc) - if err != nil { - return nil, err - } - - dirStruct := protobuf.Directory{} - err = proto.Unmarshal(data, &dirStruct) - if err != nil { - return nil, err - } - - pathParts := strings.SplitN(remainingPath, "/", 2) - entryName := pathParts[0] - var entry *protobuf.Entrypoint - var exists bool - for _, dirEntry := range dirStruct.GetEntries() { - if entryName != dirEntry.GetName() { - continue - } - if exists { - // Doubled entry - reject such directory structure - // to avoid ambiguity-based attacks - return nil, ErrCorruptedLinkData - } - exists = true - entry = dirEntry.GetEp() - } - if !exists { - return nil, ErrNotFound - } - - if len(pathParts) == 1 { - // Found the entry, no need to descend any further, only dereference the link - entry, err = DereferenceLink(ctx, d.BE, entry, d.MaxLinkRedirects, currentTime) - if err != nil { - return nil, err - } - return entry, nil - } - - return d.findEntrypointInDir(ctx, entry, pathParts[1], currentTime) -} - -func (d *CinodeFS) currentTime() time.Time { - if d.CurrentTimeF != nil { - return d.CurrentTimeF() - } - return time.Now() -} diff --git a/pkg/structure/directory.go b/pkg/structure/directory.go deleted file mode 100644 index b127fd1..0000000 --- a/pkg/structure/directory.go +++ /dev/null @@ -1,280 +0,0 @@ -/* -Copyright © 2023 Bartłomiej Święcki (byo) - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package structure - -import ( - "bytes" - "context" - "errors" - "fmt" - "html/template" - "io" - "io/fs" - "mime" - "net/http" - "path" - "path/filepath" - "sort" - - _ "embed" - - "github.com/cinode/go/pkg/blenc" - "github.com/cinode/go/pkg/blobtypes" - "github.com/cinode/go/pkg/common" - "github.com/cinode/go/pkg/structure/internal/protobuf" - "github.com/cinode/go/pkg/utilities/golang" - "golang.org/x/exp/slog" - "google.golang.org/protobuf/proto" -) - -const ( - CinodeDirMimeType = "application/cinode-dir" -) - -var ( - ErrNotFound = blenc.ErrNotFound - ErrNotADirectory = errors.New("entry is not a directory") - ErrNotAFile = errors.New("entry is not a file") -) - -func UploadStaticDirectory(ctx context.Context, log *slog.Logger, fsys fs.FS, be blenc.BE) (*protobuf.Entrypoint, error) { - c := dirCompiler{ - ctx: ctx, - fsys: fsys, - be: be, - log: log, - } - - return c.compilePath(ctx, ".") -} - -type headWriter struct { - limit int - data []byte -} - -func newHeadWriter(limit int) headWriter { - return headWriter{ - limit: limit, - data: make([]byte, limit), - } -} - -func (h *headWriter) Write(b []byte) (int, error) { - if len(h.data) >= h.limit { - return len(b), nil - } - - if len(h.data)+len(b) > h.limit { - h.data = append(h.data, b[:h.limit-len(h.data)]...) - return len(b), nil - } - - h.data = append(h.data, b...) - return len(b), nil -} - -type dirCompiler struct { - ctx context.Context - fsys fs.FS - be blenc.BE - log *slog.Logger -} - -func (d *dirCompiler) compilePath(ctx context.Context, path string) (*protobuf.Entrypoint, error) { - st, err := fs.Stat(d.fsys, path) - if err != nil { - d.log.DebugContext(ctx, "failed to stat path", "path", path, "err", err) - return nil, fmt.Errorf("couldn't check path: %w", err) - } - - if st.IsDir() { - return d.compileDir(ctx, path) - } - - if st.Mode().IsRegular() { - return d.compileFile(ctx, path) - } - - d.log.ErrorContext(ctx, "path is neither dir nor a regular file", "path", path) - return nil, fmt.Errorf("neither dir nor a regular file: %v", path) -} - -// UploadStaticBlob uploads blob to the associated datastore and returns entrypoint to that file -// -// if mimeType is an empty string, it will be guessed from the content defaulting to -func UploadStaticBlob(ctx context.Context, be blenc.BE, r io.Reader, mimeType string, log *slog.Logger) (*protobuf.Entrypoint, error) { - // Use the dataHead to store first 512 bytes of data into a buffer while uploading it to the blenc layer - // This buffer may then be used to detect the mime type - dataHead := newHeadWriter(512) - - bn, key, _, err := be.Create(context.Background(), blobtypes.Static, io.TeeReader(r, &dataHead)) - if err != nil { - log.ErrorContext(ctx, "failed to upload static file", "err", err) - return nil, err - } - - log.DebugContext(ctx, "static file uploaded successfully") - - if mimeType == "" { - mimeType = http.DetectContentType(dataHead.data) - log.DebugContext(ctx, "automatically detected content type", "contentType", mimeType) - } - - return &protobuf.Entrypoint{ - BlobName: bn.Bytes(), - KeyInfo: &protobuf.KeyInfo{Key: key.Bytes()}, - MimeType: mimeType, - }, nil -} - -func (d *dirCompiler) compileFile(ctx context.Context, path string) (*protobuf.Entrypoint, error) { - d.log.InfoContext(ctx, "compiling file", "path", path) - fl, err := d.fsys.Open(path) - if err != nil { - d.log.ErrorContext(ctx, "failed to open file", "path", path, "err", err) - return nil, fmt.Errorf("couldn't open file %v: %w", path, err) - } - defer fl.Close() - - ep, err := UploadStaticBlob( - ctx, - d.be, - fl, - mime.TypeByExtension(filepath.Ext(path)), - d.log.With("path", path), - ) - if err != nil { - return nil, fmt.Errorf("failed to upload file %v: %w", path, err) - } - - return ep, nil -} - -func (d *dirCompiler) compileDir(ctx context.Context, p string) (*protobuf.Entrypoint, error) { - fileList, err := fs.ReadDir(d.fsys, p) - if err != nil { - d.log.ErrorContext(ctx, "couldn't read contents of dir", "path", p, "err", err) - return nil, fmt.Errorf("couldn't read contents of dir %v: %w", p, err) - } - - dir := StaticDir{} - for _, e := range fileList { - subPath := path.Join(p, e.Name()) - - ep, err := d.compilePath(ctx, subPath) - if err != nil { - return nil, err - } - - dir.SetEntry(e.Name(), ep) - } - - ep, err := dir.GenerateEntrypoint(context.Background(), d.be) - if err != nil { - d.log.ErrorContext(ctx, "failed to serialize directory", "path", p, "err", err) - return nil, fmt.Errorf("can not serialize directory %v: %w", p, err) - } - - bn, _ := common.BlobNameFromBytes(ep.BlobName) - d.log.DebugContext(ctx, - "directory uploaded successfully", "path", p, - "blobName", bn.String(), - ) - return ep, nil -} - -type StaticDir struct { - entries map[string]*protobuf.Entrypoint -} - -func (s *StaticDir) SetEntry(name string, ep *protobuf.Entrypoint) { - if s.entries == nil { - s.entries = map[string]*protobuf.Entrypoint{} - } - s.entries[name] = ep -} - -//go:embed templates/dir.html -var _dirIndexTemplateStr string -var dirIndexTemplate = golang.Must( - template.New("dir"). - Funcs(template.FuncMap{ - "isDir": func(entry *protobuf.Entrypoint) bool { - return entry.MimeType == CinodeDirMimeType - }, - }). - Parse(_dirIndexTemplateStr), -) - -func (s *StaticDir) GenerateIndex(ctx context.Context, log *slog.Logger, indexName string, be blenc.BE) error { - buf := bytes.NewBuffer(nil) - err := dirIndexTemplate.Execute(buf, map[string]any{ - "entries": s.getProtobufData().GetEntries(), - "indexName": indexName, - }) - if err != nil { - return err - } - - ep, err := UploadStaticBlob(ctx, be, bytes.NewReader(buf.Bytes()), "text/html", log) - if err != nil { - return err - } - - s.entries[indexName] = ep - return nil -} - -func (s *StaticDir) getProtobufData() *protobuf.Directory { - // Convert to protobuf format - protoData := protobuf.Directory{ - Entries: make([]*protobuf.Directory_Entry, 0, len(s.entries)), - } - for name, ep := range s.entries { - protoData.Entries = append(protoData.Entries, &protobuf.Directory_Entry{ - Name: name, - Ep: ep, - }) - } - - // Sort by name - sort.Slice(protoData.Entries, func(i, j int) bool { - return protoData.Entries[i].Name < protoData.Entries[j].Name - }) - - return &protoData -} - -func (s *StaticDir) GenerateEntrypoint(ctx context.Context, be blenc.BE) (*protobuf.Entrypoint, error) { - // TODO: Introduce various directory split strategies - data, err := proto.Marshal(s.getProtobufData()) - if err != nil { - return nil, err - } - - bn, key, _, err := be.Create(context.Background(), blobtypes.Static, bytes.NewReader(data)) - if err != nil { - return nil, err - } - - return &protobuf.Entrypoint{ - BlobName: bn.Bytes(), - KeyInfo: &protobuf.KeyInfo{Key: key.Bytes()}, - MimeType: CinodeDirMimeType, - }, nil -} diff --git a/pkg/structure/http.go b/pkg/structure/http.go deleted file mode 100644 index 176ec59..0000000 --- a/pkg/structure/http.go +++ /dev/null @@ -1,72 +0,0 @@ -/* -Copyright © 2023 Bartłomiej Święcki (byo) - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package structure - -import ( - "errors" - "io" - "log" - "net/http" - "strings" -) - -type HTTPHandler struct { - FS *CinodeFS - IndexFile string -} - -func (h *HTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if r.Method != "GET" { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - path := r.URL.Path - if strings.HasSuffix(path, "/") { - path += h.IndexFile - } - path = strings.TrimPrefix(path, "/") - - fileEP, err := h.FS.FindEntrypoint(r.Context(), path) - switch { - case errors.Is(err, ErrNotFound): - http.NotFound(w, r) - return - case err != nil: - log.Println("Error serving request:", err) - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - if fileEP.MimeType == CinodeDirMimeType { - http.Redirect(w, r, r.URL.Path+"/", http.StatusPermanentRedirect) - return - } - - w.Header().Set("Content-Type", fileEP.GetMimeType()) - rc, err := h.FS.OpenContent(r.Context(), fileEP) - if err != nil { - log.Printf("Error sending file: %v", err) - } - defer rc.Close() - - _, err = io.Copy(w, rc) - if err != nil { - log.Printf("Error sending file: %v", err) - } - -} diff --git a/pkg/structure/link.go b/pkg/structure/link.go deleted file mode 100644 index 92593e5..0000000 --- a/pkg/structure/link.go +++ /dev/null @@ -1,138 +0,0 @@ -/* -Copyright © 2023 Bartłomiej Święcki (byo) - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package structure - -import ( - "bytes" - "context" - "errors" - "io" - "time" - - "github.com/cinode/go/pkg/blenc" - "github.com/cinode/go/pkg/blobtypes" - "github.com/cinode/go/pkg/common" - "github.com/cinode/go/pkg/structure/internal/protobuf" -) - -var ( - ErrMaxRedirectsReached = errors.New("maximum limit of dynamic link redirects reached") - ErrCorruptedLinkData = errors.New("corrupted link data") - ErrCorruptedDirectoryData = errors.New("corrupted directory data") - ErrInvalidEntrypoint = protobuf.ErrInvalidEntrypoint - ErrInvalidEntrypointTime = protobuf.ErrInvalidEntrypointTime -) - -func CreateLink(ctx context.Context, be blenc.BE, ep *protobuf.Entrypoint) (*protobuf.Entrypoint, *protobuf.WriterInfo, error) { - epBytes, err := ep.ToBytes() - if err != nil { - return nil, nil, err - } - - name, key, authInfo, err := be.Create(ctx, blobtypes.DynamicLink, bytes.NewReader(epBytes)) - if err != nil { - return nil, nil, err - } - - return &protobuf.Entrypoint{ - BlobName: name.Bytes(), - KeyInfo: &protobuf.KeyInfo{ - Key: key.Bytes(), - }, - }, &protobuf.WriterInfo{ - BlobName: name.Bytes(), - Key: key.Bytes(), - AuthInfo: authInfo, - }, nil -} - -func UpdateLink(ctx context.Context, be blenc.BE, wi *protobuf.WriterInfo, ep *protobuf.Entrypoint) (*protobuf.Entrypoint, error) { - epBytes, err := ep.ToBytes() - if err != nil { - return nil, err - } - - bn, err := common.BlobNameFromBytes(wi.BlobName) - if err != nil { - return nil, err - } - - err = be.Update( - ctx, - bn, - wi.AuthInfo, - common.BlobKeyFromBytes(wi.Key), - bytes.NewReader(epBytes), - ) - if err != nil { - return nil, err - } - - return &protobuf.Entrypoint{ - BlobName: wi.BlobName, - KeyInfo: &protobuf.KeyInfo{ - Key: wi.Key, - }, - }, nil -} - -func DereferenceLink( - ctx context.Context, - be blenc.BE, - link *protobuf.Entrypoint, - maxRedirects int, - currentTime time.Time, -) ( - *protobuf.Entrypoint, - error, -) { - bn, key, err := link.ValidateAndParse(currentTime) - if err != nil { - return nil, err - } - - for bn.Type() == blobtypes.DynamicLink { - if maxRedirects == 0 { - return nil, ErrMaxRedirectsReached - } - maxRedirects-- - - rc, err := be.Open(ctx, bn, key) - if err != nil { - return nil, err - } - defer rc.Close() - - // TODO: Constrain the buffer size - data, err := io.ReadAll(rc) - if err != nil { - return nil, err - } - - link, err = protobuf.EntryPointFromBytes(data) - if err != nil { - return nil, err - } - - bn, key, err = link.ValidateAndParse(time.Now()) - if err != nil { - return nil, err - } - } - - return link, nil -} diff --git a/pkg/structure/templates/dir.html b/pkg/structure/templates/dir.html deleted file mode 100644 index f5ad40a..0000000 --- a/pkg/structure/templates/dir.html +++ /dev/null @@ -1,73 +0,0 @@ -{{/* -Copyright © 2023 Bartłomiej Święcki (byo) - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - - - - - Directory Listing - - - - -

Directory Listing

- - - - - - - {{range .entries}}{{if isDir .Ep}}{{if ne .Name $.indexName}} - - - - - - {{end}}{{end}}{{end}} - {{range .entries}}{{if not (isDir .Ep) }}{{if ne .Name $.indexName}} - - - - - - {{end}}{{end}}{{end}} -
DirNameMimeType
[DIR]{{.Name}}{{.Ep.MimeType}}
{{.Name}}{{.Ep.MimeType}}
- - - diff --git a/testvectors/testblobs/base.go b/testvectors/testblobs/base.go index 89c641d..78da468 100644 --- a/testvectors/testblobs/base.go +++ b/testvectors/testblobs/base.go @@ -23,8 +23,8 @@ import ( "net/http" "net/url" + "github.com/cinode/go/pkg/cinodefs" "github.com/cinode/go/pkg/common" - "github.com/cinode/go/pkg/structure/graph" "github.com/jbenet/go-base58" ) @@ -99,8 +99,8 @@ func (s *TestBlob) Get(baseUrl string) ([]byte, error) { return body, nil } -func (s *TestBlob) Entrypoint() *graph.Entrypoint { - return graph.EntrypointFromBlobNameAndKey( +func (s *TestBlob) Entrypoint() *cinodefs.Entrypoint { + return cinodefs.EntrypointFromBlobNameAndKey( s.BlobName, s.EncryptionKey, ) From eb27fdfb90e53b70418517a96165a68370cf5256 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Sun, 29 Oct 2023 13:05:57 +0100 Subject: [PATCH 15/29] Switch back BlobName and BlobKey to pointers This is much more natural in Go where lack of value is usually denoted through nil as opposed to default value. --- pkg/blenc/datastore.go | 14 +++--- pkg/blenc/datastore_dynamic_link.go | 18 ++++---- pkg/blenc/datastore_dynamic_link_test.go | 16 +++---- pkg/blenc/datastore_static.go | 28 ++++++------ pkg/blenc/datastore_static_test.go | 6 +-- pkg/blenc/interface.go | 10 ++--- pkg/blenc/interface_test.go | 4 +- pkg/cinodefs/context.go | 4 +- pkg/cinodefs/entrypoint.go | 8 ++-- pkg/cinodefs/internal/protobuf/protobuf.go | 8 ++-- pkg/cinodefs/writerinfo.go | 2 +- pkg/common/blob_keys.go | 12 ++--- pkg/common/blob_keys_test.go | 4 +- pkg/common/blob_name.go | 44 ++++++++++--------- pkg/common/blob_name_test.go | 2 +- pkg/datastore/datastore.go | 8 ++-- pkg/datastore/datastore_dynamic_link.go | 8 ++-- pkg/datastore/datastore_static.go | 4 +- pkg/datastore/datastore_test.go | 22 +++++----- pkg/datastore/interface.go | 10 ++--- pkg/datastore/multi_source.go | 12 ++--- pkg/datastore/multi_source_test.go | 6 +-- pkg/datastore/storage.go | 10 ++--- pkg/datastore/storage_filesystem.go | 14 +++--- pkg/datastore/storage_memory.go | 10 ++--- pkg/datastore/storage_raw_filesystem.go | 8 ++-- pkg/datastore/storage_test.go | 2 +- pkg/datastore/utils_autogen_for_test.go | 6 +-- pkg/datastore/utils_for_test.go | 6 +-- pkg/datastore/webconnector.go | 12 ++--- pkg/datastore/webinterface.go | 8 ++-- pkg/datastore/webinterface_test.go | 4 +- pkg/internal/blobtypes/dynamiclink/public.go | 10 ++--- .../blobtypes/dynamiclink/public_test.go | 6 +-- .../blobtypes/dynamiclink/publisher.go | 12 ++--- .../blobtypes/dynamiclink/publisher_test.go | 5 +-- .../utilities/cipherfactory/cipher_factory.go | 6 +-- .../utilities/cipherfactory/generator.go | 13 +++--- testvectors/testblobs/base.go | 9 ++-- 39 files changed, 195 insertions(+), 196 deletions(-) diff --git a/pkg/blenc/datastore.go b/pkg/blenc/datastore.go index 78cc2cf..d23ea4b 100644 --- a/pkg/blenc/datastore.go +++ b/pkg/blenc/datastore.go @@ -50,7 +50,7 @@ type beDatastore struct { newSecureFifo secureFifoGenerator } -func (be *beDatastore) Open(ctx context.Context, name common.BlobName, key common.BlobKey) (io.ReadCloser, error) { +func (be *beDatastore) Open(ctx context.Context, name *common.BlobName, key *common.BlobKey) (io.ReadCloser, error) { switch name.Type() { case blobtypes.Static: return be.openStatic(ctx, name, key) @@ -65,8 +65,8 @@ func (be *beDatastore) Create( blobType common.BlobType, r io.Reader, ) ( - common.BlobName, - common.BlobKey, + *common.BlobName, + *common.BlobKey, AuthInfo, error, ) { @@ -76,10 +76,10 @@ func (be *beDatastore) Create( case blobtypes.DynamicLink: return be.createDynamicLink(ctx, r) } - return common.BlobName{}, common.BlobKey{}, nil, blobtypes.ErrUnknownBlobType + return nil, nil, nil, blobtypes.ErrUnknownBlobType } -func (be *beDatastore) Update(ctx context.Context, name common.BlobName, authInfo AuthInfo, key common.BlobKey, r io.Reader) error { +func (be *beDatastore) Update(ctx context.Context, name *common.BlobName, authInfo AuthInfo, key *common.BlobKey, r io.Reader) error { switch name.Type() { case blobtypes.Static: return be.updateStatic(ctx, name, authInfo, key, r) @@ -89,10 +89,10 @@ func (be *beDatastore) Update(ctx context.Context, name common.BlobName, authInf return blobtypes.ErrUnknownBlobType } -func (be *beDatastore) Exists(ctx context.Context, name common.BlobName) (bool, error) { +func (be *beDatastore) Exists(ctx context.Context, name *common.BlobName) (bool, error) { return be.ds.Exists(ctx, name) } -func (be *beDatastore) Delete(ctx context.Context, name common.BlobName) error { +func (be *beDatastore) Delete(ctx context.Context, name *common.BlobName) error { return be.ds.Delete(ctx, name) } diff --git a/pkg/blenc/datastore_dynamic_link.go b/pkg/blenc/datastore_dynamic_link.go index b78de2f..3d754c0 100644 --- a/pkg/blenc/datastore_dynamic_link.go +++ b/pkg/blenc/datastore_dynamic_link.go @@ -35,8 +35,8 @@ var ( func (be *beDatastore) openDynamicLink( ctx context.Context, - name common.BlobName, - key common.BlobKey, + name *common.BlobName, + key *common.BlobKey, ) ( io.ReadCloser, error, @@ -75,8 +75,8 @@ func (be *beDatastore) createDynamicLink( ctx context.Context, r io.Reader, ) ( - common.BlobName, - common.BlobKey, + *common.BlobName, + *common.BlobKey, AuthInfo, error, ) { @@ -84,19 +84,19 @@ func (be *beDatastore) createDynamicLink( dl, err := dynamiclink.Create(be.rand) if err != nil { - return common.BlobName{}, common.BlobKey{}, nil, err + return nil, nil, nil, err } pr, encryptionKey, err := dl.UpdateLinkData(r, version) if err != nil { - return common.BlobName{}, common.BlobKey{}, nil, err + return nil, nil, nil, err } // Send update packet bn := dl.BlobName() err = be.ds.Update(ctx, bn, pr.GetPublicDataReader()) if err != nil { - return common.BlobName{}, common.BlobKey{}, nil, err + return nil, nil, nil, err } return bn, @@ -107,9 +107,9 @@ func (be *beDatastore) createDynamicLink( func (be *beDatastore) updateDynamicLink( ctx context.Context, - name common.BlobName, + name *common.BlobName, authInfo AuthInfo, - key common.BlobKey, + key *common.BlobKey, r io.Reader, ) error { newVersion := be.generateVersion() diff --git a/pkg/blenc/datastore_dynamic_link_test.go b/pkg/blenc/datastore_dynamic_link_test.go index c1942d5..f124b6b 100644 --- a/pkg/blenc/datastore_dynamic_link_test.go +++ b/pkg/blenc/datastore_dynamic_link_test.go @@ -34,18 +34,18 @@ import ( type dsWrapper struct { datastore.DS - openFn func(ctx context.Context, name common.BlobName) (io.ReadCloser, error) - updateFn func(ctx context.Context, name common.BlobName, r io.Reader) error + openFn func(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) + updateFn func(ctx context.Context, name *common.BlobName, r io.Reader) error } -func (w *dsWrapper) Open(ctx context.Context, name common.BlobName) (io.ReadCloser, error) { +func (w *dsWrapper) Open(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { if w.openFn != nil { return w.openFn(ctx, name) } return w.DS.Open(ctx, name) } -func (w *dsWrapper) Update(ctx context.Context, name common.BlobName, r io.Reader) error { +func (w *dsWrapper) Update(ctx context.Context, name *common.BlobName, r io.Reader) error { if w.updateFn != nil { return w.updateFn(ctx, name, r) } @@ -68,7 +68,7 @@ func TestDynamicLinkErrors(t *testing.T) { t.Run("handle error while opening blob", func(t *testing.T) { injectedErr := errors.New("test") - dsw.openFn = func(ctx context.Context, name common.BlobName) (io.ReadCloser, error) { return nil, injectedErr } + dsw.openFn = func(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { return nil, injectedErr } rc, err := be.Open(context.Background(), bn, key) require.ErrorIs(t, err, injectedErr) @@ -84,7 +84,7 @@ func TestDynamicLinkErrors(t *testing.T) { t.Run(fmt.Sprintf("error at byte %d", i), func(t *testing.T) { - dsw.openFn = func(ctx context.Context, name common.BlobName) (io.ReadCloser, error) { + dsw.openFn = func(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { origRC, err := dsw.DS.Open(ctx, name) require.NoError(t, err) @@ -135,7 +135,7 @@ func TestDynamicLinkErrors(t *testing.T) { t.Run("fail to store new dynamic link blob", func(t *testing.T) { injectedErr := errors.New("test") - dsw.updateFn = func(ctx context.Context, name common.BlobName, r io.Reader) error { return injectedErr } + dsw.updateFn = func(ctx context.Context, name *common.BlobName, r io.Reader) error { return injectedErr } bn, key, ai, err := be.Create(context.Background(), blobtypes.DynamicLink, bytes.NewReader(nil)) require.ErrorIs(t, err, injectedErr) @@ -152,7 +152,7 @@ func TestDynamicLinkErrors(t *testing.T) { bn, key, ai, err := be.Create(context.Background(), blobtypes.DynamicLink, bytes.NewReader(nil)) require.NoError(t, err) - dsw.updateFn = func(ctx context.Context, name common.BlobName, r io.Reader) error { return injectedErr } + dsw.updateFn = func(ctx context.Context, name *common.BlobName, r io.Reader) error { return injectedErr } err = be.Update(context.Background(), bn, ai, key, bytes.NewReader(nil)) require.ErrorIs(t, err, injectedErr) diff --git a/pkg/blenc/datastore_static.go b/pkg/blenc/datastore_static.go index 4dcc052..71e88df 100644 --- a/pkg/blenc/datastore_static.go +++ b/pkg/blenc/datastore_static.go @@ -32,7 +32,7 @@ var ( ErrCanNotUpdateStaticBlob = errors.New("blob update is not supported for static blobs") ) -func (be *beDatastore) openStatic(ctx context.Context, name common.BlobName, key common.BlobKey) (io.ReadCloser, error) { +func (be *beDatastore) openStatic(ctx context.Context, name *common.BlobName, key *common.BlobKey) (io.ReadCloser, error) { rc, err := be.ds.Open(ctx, name) if err != nil { @@ -67,27 +67,27 @@ func (be *beDatastore) createStatic( ctx context.Context, r io.Reader, ) ( - common.BlobName, - common.BlobKey, + *common.BlobName, + *common.BlobKey, AuthInfo, error, ) { tempWriteBufferPlain, err := be.newSecureFifo() if err != nil { - return common.BlobName{}, common.BlobKey{}, nil, err + return nil, nil, nil, err } defer tempWriteBufferPlain.Close() tempWriteBufferEncrypted, err := be.newSecureFifo() if err != nil { - return common.BlobName{}, common.BlobKey{}, nil, err + return nil, nil, nil, err } defer tempWriteBufferEncrypted.Close() keyGenerator := cipherfactory.NewKeyGenerator(blobtypes.Static) _, err = io.Copy(tempWriteBufferPlain, io.TeeReader(r, keyGenerator)) if err != nil { - return common.BlobName{}, common.BlobKey{}, nil, err + return nil, nil, nil, err } key := keyGenerator.Generate() @@ -95,7 +95,7 @@ func (be *beDatastore) createStatic( rClone, err := tempWriteBufferPlain.Done() // rClone will allow re-reading the source data if err != nil { - return common.BlobName{}, common.BlobKey{}, nil, err + return nil, nil, nil, err } defer rClone.Close() @@ -109,30 +109,30 @@ func (be *beDatastore) createStatic( ), ) if err != nil { - return common.BlobName{}, common.BlobKey{}, nil, err + return nil, nil, nil, err } _, err = io.Copy(encWriter, rClone) if err != nil { - return common.BlobName{}, common.BlobKey{}, nil, err + return nil, nil, nil, err } encReader, err := tempWriteBufferEncrypted.Done() if err != nil { - return common.BlobName{}, common.BlobKey{}, nil, err + return nil, nil, nil, err } defer encReader.Close() // Generate blob name from the encrypted data name, err := common.BlobNameFromHashAndType(blobNameHasher.Sum(nil), blobtypes.Static) if err != nil { - return common.BlobName{}, common.BlobKey{}, nil, err + return nil, nil, nil, err } // Send encrypted blob into the datastore err = be.ds.Update(ctx, name, encReader) if err != nil { - return common.BlobName{}, common.BlobKey{}, nil, err + return nil, nil, nil, err } return name, key, nil, nil @@ -140,9 +140,9 @@ func (be *beDatastore) createStatic( func (be *beDatastore) updateStatic( ctx context.Context, - name common.BlobName, + name *common.BlobName, authInfo AuthInfo, - key common.BlobKey, + key *common.BlobKey, r io.Reader, ) error { return ErrCanNotUpdateStaticBlob diff --git a/pkg/blenc/datastore_static_test.go b/pkg/blenc/datastore_static_test.go index b521e84..d0da6e6 100644 --- a/pkg/blenc/datastore_static_test.go +++ b/pkg/blenc/datastore_static_test.go @@ -69,7 +69,7 @@ func TestStaticErrorTruncatedDatastore(t *testing.T) { t.Run("handle error while opening blob", func(t *testing.T) { injectedErr := errors.New("test") - dsw.openFn = func(ctx context.Context, name common.BlobName) (io.ReadCloser, error) { return nil, injectedErr } + dsw.openFn = func(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { return nil, injectedErr } rc, err := be.Open(context.Background(), bn, key) require.ErrorIs(t, err, injectedErr) @@ -80,7 +80,7 @@ func TestStaticErrorTruncatedDatastore(t *testing.T) { t.Run("handle error while opening blob", func(t *testing.T) { injectedErr := errors.New("test") - dsw.openFn = func(ctx context.Context, name common.BlobName) (io.ReadCloser, error) { return nil, injectedErr } + dsw.openFn = func(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { return nil, injectedErr } rc, err := be.Open(context.Background(), bn, key) require.ErrorIs(t, err, injectedErr) @@ -267,7 +267,7 @@ func TestStaticErrorTruncatedDatastore(t *testing.T) { }, nil } - dsw.updateFn = func(ctx context.Context, name common.BlobName, r io.Reader) error { return injectedErr } + dsw.updateFn = func(ctx context.Context, name *common.BlobName, r io.Reader) error { return injectedErr } bn, key, ai, err := be.Create(context.Background(), blobtypes.Static, bytes.NewReader(nil)) require.ErrorIs(t, err, injectedErr) diff --git a/pkg/blenc/interface.go b/pkg/blenc/interface.go index 65ccb51..4124d1d 100644 --- a/pkg/blenc/interface.go +++ b/pkg/blenc/interface.go @@ -39,23 +39,23 @@ type BE interface { // // If returned error is not nil, the reader must be nil. Otherwise it is required to // close the reader once done working with it. - Open(ctx context.Context, name common.BlobName, key common.BlobKey) (io.ReadCloser, error) + Open(ctx context.Context, name *common.BlobName, key *common.BlobKey) (io.ReadCloser, error) // Create completely new blob with given dataset, as a result, the blob name and optional // AuthInfo that allows blob's update is returned - Create(ctx context.Context, blobType common.BlobType, r io.Reader) (common.BlobName, common.BlobKey, AuthInfo, error) + Create(ctx context.Context, blobType common.BlobType, r io.Reader) (*common.BlobName, *common.BlobKey, AuthInfo, error) // Update updates given blob type with new data, // The update must happen within a single blob name (i.e. it can not end up with blob with different name) // and may not be available for certain blob types such as static blobs. // A valid auth info is necessary to ensure a correct new content can be created - Update(ctx context.Context, name common.BlobName, ai AuthInfo, key common.BlobKey, r io.Reader) error + Update(ctx context.Context, name *common.BlobName, ai AuthInfo, key *common.BlobKey, r io.Reader) error // Exists does check whether blob of given name exists. It forwards the call // to underlying datastore. - Exists(ctx context.Context, name common.BlobName) (bool, error) + Exists(ctx context.Context, name *common.BlobName) (bool, error) // Delete tries to remove blob with given name. It forwards the call to // underlying datastore. - Delete(ctx context.Context, name common.BlobName) error + Delete(ctx context.Context, name *common.BlobName) error } diff --git a/pkg/blenc/interface_test.go b/pkg/blenc/interface_test.go index 1f3f174..ff2f472 100644 --- a/pkg/blenc/interface_test.go +++ b/pkg/blenc/interface_test.go @@ -265,7 +265,7 @@ func (s *BlencTestSuite) TestInvalidBlobTypes() { rc, err := s.be.Open( context.Background(), invalidBlobName, - common.BlobKey{}, + nil, ) s.Require().ErrorIs(err, blobtypes.ErrUnknownBlobType) s.Require().Nil(rc) @@ -276,7 +276,7 @@ func (s *BlencTestSuite) TestInvalidBlobTypes() { context.Background(), invalidBlobName, AuthInfo{}, - common.BlobKey{}, + nil, bytes.NewReader(nil), ) s.Require().ErrorIs(err, blobtypes.ErrUnknownBlobType) diff --git a/pkg/cinodefs/context.go b/pkg/cinodefs/context.go index 4606502..bfa1135 100644 --- a/pkg/cinodefs/context.go +++ b/pkg/cinodefs/context.go @@ -50,10 +50,10 @@ type graphContext struct { func (c *graphContext) keyFromEntrypoint( ctx context.Context, ep *Entrypoint, -) (common.BlobKey, error) { +) (*common.BlobKey, error) { if ep.ep.KeyInfo == nil || ep.ep.KeyInfo.Key == nil { - return common.BlobKey{}, ErrMissingKeyInfo + return nil, ErrMissingKeyInfo } return common.BlobKeyFromBytes(ep.ep.GetKeyInfo().GetKey()), nil } diff --git a/pkg/cinodefs/entrypoint.go b/pkg/cinodefs/entrypoint.go index 4a0314a..c8dcfcc 100644 --- a/pkg/cinodefs/entrypoint.go +++ b/pkg/cinodefs/entrypoint.go @@ -41,7 +41,7 @@ var ( type Entrypoint struct { ep protobuf.Entrypoint - bn common.BlobName + bn *common.BlobName } func EntrypointFromString(s string) (*Entrypoint, error) { @@ -103,11 +103,11 @@ func expandEntrypointProto(ep *Entrypoint) error { return nil } -func EntrypointFromBlobNameAndKey(bn common.BlobName, key common.BlobKey) *Entrypoint { +func EntrypointFromBlobNameAndKey(bn *common.BlobName, key *common.BlobKey) *Entrypoint { return setEntrypointBlobNameAndKey(bn, key, &Entrypoint{}) } -func setEntrypointBlobNameAndKey(bn common.BlobName, key common.BlobKey, ep *Entrypoint) *Entrypoint { +func setEntrypointBlobNameAndKey(bn *common.BlobName, key *common.BlobKey, ep *Entrypoint) *Entrypoint { ep.bn = bn ep.ep.BlobName = bn.Bytes() ep.ep.KeyInfo = &protobuf.KeyInfo{Key: key.Bytes()} @@ -122,7 +122,7 @@ func (e *Entrypoint) Bytes() []byte { return golang.Must(proto.Marshal(&e.ep)) } -func (e *Entrypoint) BlobName() common.BlobName { +func (e *Entrypoint) BlobName() *common.BlobName { return e.bn } diff --git a/pkg/cinodefs/internal/protobuf/protobuf.go b/pkg/cinodefs/internal/protobuf/protobuf.go index b585a8d..79593d3 100644 --- a/pkg/cinodefs/internal/protobuf/protobuf.go +++ b/pkg/cinodefs/internal/protobuf/protobuf.go @@ -74,18 +74,18 @@ func (ep *Entrypoint) Validate(currentTime time.Time) error { } func (ep *Entrypoint) ValidateAndParse(currentTime time.Time) ( - common.BlobName, - common.BlobKey, + *common.BlobName, + *common.BlobKey, error, ) { err := ep.Validate(currentTime) if err != nil { - return common.BlobName{}, common.BlobKey{}, err + return nil, nil, err } bn, err := common.BlobNameFromBytes(ep.BlobName) if err != nil { - return common.BlobName{}, common.BlobKey{}, err + return nil, nil, err } key := common.BlobKeyFromBytes(ep.GetKeyInfo().GetKey()) diff --git a/pkg/cinodefs/writerinfo.go b/pkg/cinodefs/writerinfo.go index f69ae71..ca6bfaf 100644 --- a/pkg/cinodefs/writerinfo.go +++ b/pkg/cinodefs/writerinfo.go @@ -68,7 +68,7 @@ func WriterInfoFromBytes(b []byte) (*WriterInfo, error) { return &wi, nil } -func writerInfoFromBlobNameKeyAndAuthInfo(bn common.BlobName, key common.BlobKey, ai []byte) *WriterInfo { +func writerInfoFromBlobNameKeyAndAuthInfo(bn *common.BlobName, key *common.BlobKey, ai []byte) *WriterInfo { return &WriterInfo{ wi: protobuf.WriterInfo{ BlobName: bn.Bytes(), diff --git a/pkg/common/blob_keys.go b/pkg/common/blob_keys.go index 5518874..60e1adb 100644 --- a/pkg/common/blob_keys.go +++ b/pkg/common/blob_keys.go @@ -30,13 +30,13 @@ func copyBytes(b []byte) []byte { // Key with cipher type type BlobKey struct{ key []byte } -func BlobKeyFromBytes(key []byte) BlobKey { return BlobKey{key: copyBytes(key)} } -func (k BlobKey) Bytes() []byte { return copyBytes(k.key) } -func (k BlobKey) Equal(k2 BlobKey) bool { return subtle.ConstantTimeCompare(k.key, k2.key) == 1 } +func BlobKeyFromBytes(key []byte) *BlobKey { return &BlobKey{key: copyBytes(key)} } +func (k *BlobKey) Bytes() []byte { return copyBytes(k.key) } +func (k *BlobKey) Equal(k2 *BlobKey) bool { return subtle.ConstantTimeCompare(k.key, k2.key) == 1 } // IV type BlobIV struct{ iv []byte } -func BlobIVFromBytes(iv []byte) BlobIV { return BlobIV{iv: copyBytes(iv)} } -func (i BlobIV) Bytes() []byte { return copyBytes(i.iv) } -func (i BlobIV) Equal(i2 BlobIV) bool { return subtle.ConstantTimeCompare(i.iv, i2.iv) == 1 } +func BlobIVFromBytes(iv []byte) *BlobIV { return &BlobIV{iv: copyBytes(iv)} } +func (i *BlobIV) Bytes() []byte { return copyBytes(i.iv) } +func (i *BlobIV) Equal(i2 *BlobIV) bool { return subtle.ConstantTimeCompare(i.iv, i2.iv) == 1 } diff --git a/pkg/common/blob_keys_test.go b/pkg/common/blob_keys_test.go index 171f7ae..28b32df 100644 --- a/pkg/common/blob_keys_test.go +++ b/pkg/common/blob_keys_test.go @@ -27,7 +27,7 @@ func TestBlobKey(t *testing.T) { key := BlobKeyFromBytes(keyBytes) require.Equal(t, keyBytes, key.Bytes()) require.True(t, key.Equal(BlobKeyFromBytes(keyBytes))) - require.Nil(t, BlobKey{}.Bytes()) + require.Nil(t, new(BlobKey).Bytes()) } func TestBlobIV(t *testing.T) { @@ -35,5 +35,5 @@ func TestBlobIV(t *testing.T) { iv := BlobIVFromBytes(ivBytes) require.Equal(t, ivBytes, iv.Bytes()) require.True(t, iv.Equal(BlobIVFromBytes(ivBytes))) - require.Nil(t, BlobKey{}.Bytes()) + require.Nil(t, new(BlobKey).Bytes()) } diff --git a/pkg/common/blob_name.go b/pkg/common/blob_name.go index ea7cddf..d664c69 100644 --- a/pkg/common/blob_name.go +++ b/pkg/common/blob_name.go @@ -33,63 +33,65 @@ var ( // The type of the blob is not stored directly. Instead it is mixed // with the hash of the blob to make sure that all bytes in the blob name // are randomly distributed. -type BlobName []byte +type BlobName struct { + bn []byte +} // BlobNameFromHashAndType generates the name of a blob from some hash (e.g. sha256 of blob's content) // and given blob type -func BlobNameFromHashAndType(hash []byte, t BlobType) (BlobName, error) { +func BlobNameFromHashAndType(hash []byte, t BlobType) (*BlobName, error) { if len(hash) == 0 || len(hash) > 0x7E { - return BlobName{}, ErrInvalidBlobName + return nil, ErrInvalidBlobName } - ret := make([]byte, len(hash)+1) + bn := make([]byte, len(hash)+1) - copy(ret[1:], hash) + copy(bn[1:], hash) scrambledTypeByte := byte(t.t) for _, b := range hash { scrambledTypeByte ^= b } - ret[0] = scrambledTypeByte + bn[0] = scrambledTypeByte - return BlobName(ret), nil + return &BlobName{bn: bn}, nil } // BlobNameFromString decodes base58-encoded string into blob name -func BlobNameFromString(s string) (BlobName, error) { +func BlobNameFromString(s string) (*BlobName, error) { return BlobNameFromBytes(base58.Decode(s)) } -func BlobNameFromBytes(n []byte) (BlobName, error) { +func BlobNameFromBytes(n []byte) (*BlobName, error) { if len(n) == 0 || len(n) > 0x7F { - return BlobName{}, ErrInvalidBlobName + return nil, ErrInvalidBlobName } - return BlobName(copyBytes(n)), nil + return &BlobName{bn: copyBytes(n)}, nil } // Returns base58-encoded blob name -func (b BlobName) String() string { - return base58.Encode(b) +func (b *BlobName) String() string { + return base58.Encode(b.bn) } // Extracts hash from blob name -func (b BlobName) Hash() []byte { - return b[1:] +func (b *BlobName) Hash() []byte { + return b.bn[1:] } // Extracts blob type from the name -func (b BlobName) Type() BlobType { +func (b *BlobName) Type() BlobType { ret := byte(0) - for _, by := range b { + for _, by := range b.bn { ret ^= by } return BlobType{t: ret} } -func (b BlobName) Bytes() []byte { - return copyBytes(b) +func (b *BlobName) Bytes() []byte { + return copyBytes(b.bn) } -func (b BlobName) Equal(b2 BlobName) bool { - return subtle.ConstantTimeCompare(b, b2) == 1 +func (b *BlobName) Equal(b2 *BlobName) bool { + return subtle.ConstantTimeCompare(b.bn, b2.bn) == 1 } diff --git a/pkg/common/blob_name_test.go b/pkg/common/blob_name_test.go index 56682e2..f60e8c2 100644 --- a/pkg/common/blob_name_test.go +++ b/pkg/common/blob_name_test.go @@ -43,7 +43,7 @@ func TestBlobName(t *testing.T) { bn, err := BlobNameFromHashAndType(h, bt) assert.NoError(t, err) assert.NotEmpty(t, bn) - assert.Greater(t, len(bn), len(h)) + assert.Greater(t, len(bn.bn), len(h)) assert.Equal(t, h, bn.Hash()) assert.Equal(t, bt, bn.Type()) diff --git a/pkg/datastore/datastore.go b/pkg/datastore/datastore.go index cc9591d..5767ff3 100644 --- a/pkg/datastore/datastore.go +++ b/pkg/datastore/datastore.go @@ -38,7 +38,7 @@ func (ds *datastore) Address() string { return ds.s.address() } -func (ds *datastore) Open(ctx context.Context, name common.BlobName) (io.ReadCloser, error) { +func (ds *datastore) Open(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { switch name.Type() { case blobtypes.Static: return ds.openStatic(ctx, name) @@ -49,7 +49,7 @@ func (ds *datastore) Open(ctx context.Context, name common.BlobName) (io.ReadClo } } -func (ds *datastore) Update(ctx context.Context, name common.BlobName, updateStream io.Reader) error { +func (ds *datastore) Update(ctx context.Context, name *common.BlobName, updateStream io.Reader) error { switch name.Type() { case blobtypes.Static: return ds.updateStatic(ctx, name, updateStream) @@ -60,11 +60,11 @@ func (ds *datastore) Update(ctx context.Context, name common.BlobName, updateStr } } -func (ds *datastore) Exists(ctx context.Context, name common.BlobName) (bool, error) { +func (ds *datastore) Exists(ctx context.Context, name *common.BlobName) (bool, error) { return ds.s.exists(ctx, name) } -func (ds *datastore) Delete(ctx context.Context, name common.BlobName) error { +func (ds *datastore) Delete(ctx context.Context, name *common.BlobName) error { return ds.s.delete(ctx, name) } diff --git a/pkg/datastore/datastore_dynamic_link.go b/pkg/datastore/datastore_dynamic_link.go index ed0375a..7ed3bbb 100644 --- a/pkg/datastore/datastore_dynamic_link.go +++ b/pkg/datastore/datastore_dynamic_link.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ import ( "github.com/cinode/go/pkg/internal/blobtypes/dynamiclink" ) -func (ds *datastore) openDynamicLink(ctx context.Context, name common.BlobName) (io.ReadCloser, error) { +func (ds *datastore) openDynamicLink(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { rc, err := ds.s.openReadStream(ctx, name) if err != nil { return nil, err @@ -50,7 +50,7 @@ func (ds *datastore) openDynamicLink(ctx context.Context, name common.BlobName) // read from - only for comparison func (ds *datastore) newLinkGreaterThanCurrent( ctx context.Context, - name common.BlobName, + name *common.BlobName, newLink *dynamiclink.PublicReader, ) ( bool, error, @@ -72,7 +72,7 @@ func (ds *datastore) newLinkGreaterThanCurrent( return newLink.GreaterThan(dl), nil } -func (ds *datastore) updateDynamicLink(ctx context.Context, name common.BlobName, updateStream io.Reader) error { +func (ds *datastore) updateDynamicLink(ctx context.Context, name *common.BlobName, updateStream io.Reader) error { ws, err := ds.s.openWriteStream(ctx, name) if err != nil { return err diff --git a/pkg/datastore/datastore_static.go b/pkg/datastore/datastore_static.go index bee30a9..05bb5bb 100644 --- a/pkg/datastore/datastore_static.go +++ b/pkg/datastore/datastore_static.go @@ -27,7 +27,7 @@ import ( "github.com/cinode/go/pkg/internal/utilities/validatingreader" ) -func (ds *datastore) openStatic(ctx context.Context, name common.BlobName) (io.ReadCloser, error) { +func (ds *datastore) openStatic(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { rc, err := ds.s.openReadStream(ctx, name) if err != nil { return nil, err @@ -47,7 +47,7 @@ func (ds *datastore) openStatic(ctx context.Context, name common.BlobName) (io.R }, nil } -func (ds *datastore) updateStatic(ctx context.Context, name common.BlobName, updateStream io.Reader) error { +func (ds *datastore) updateStatic(ctx context.Context, name *common.BlobName, updateStream io.Reader) error { outputStream, err := ds.s.openWriteStream(ctx, name) if err != nil { return err diff --git a/pkg/datastore/datastore_test.go b/pkg/datastore/datastore_test.go index 966cc45..6b53f8a 100644 --- a/pkg/datastore/datastore_test.go +++ b/pkg/datastore/datastore_test.go @@ -32,10 +32,10 @@ import ( type mockStore struct { fKind func() string fAddress func() string - fOpenReadStream func(ctx context.Context, name common.BlobName) (io.ReadCloser, error) - fOpenWriteStream func(ctx context.Context, name common.BlobName) (WriteCloseCanceller, error) - fExists func(ctx context.Context, name common.BlobName) (bool, error) - fDelete func(ctx context.Context, name common.BlobName) error + fOpenReadStream func(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) + fOpenWriteStream func(ctx context.Context, name *common.BlobName) (WriteCloseCanceller, error) + fExists func(ctx context.Context, name *common.BlobName) (bool, error) + fDelete func(ctx context.Context, name *common.BlobName) error } func (s *mockStore) kind() string { @@ -44,16 +44,16 @@ func (s *mockStore) kind() string { func (s *mockStore) address() string { return s.fAddress() } -func (s *mockStore) openReadStream(ctx context.Context, name common.BlobName) (io.ReadCloser, error) { +func (s *mockStore) openReadStream(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { return s.fOpenReadStream(ctx, name) } -func (s *mockStore) openWriteStream(ctx context.Context, name common.BlobName) (WriteCloseCanceller, error) { +func (s *mockStore) openWriteStream(ctx context.Context, name *common.BlobName) (WriteCloseCanceller, error) { return s.fOpenWriteStream(ctx, name) } -func (s *mockStore) exists(ctx context.Context, name common.BlobName) (bool, error) { +func (s *mockStore) exists(ctx context.Context, name *common.BlobName) (bool, error) { return s.fExists(ctx, name) } -func (s *mockStore) delete(ctx context.Context, name common.BlobName) error { +func (s *mockStore) delete(ctx context.Context, name *common.BlobName) error { return s.fDelete(ctx, name) } @@ -77,7 +77,7 @@ func TestDatastoreWriteFailure(t *testing.T) { t.Run("error on opening write stream", func(t *testing.T) { errRet := errors.New("error") ds := &datastore{s: &mockStore{ - fOpenWriteStream: func(ctx context.Context, name common.BlobName) (WriteCloseCanceller, error) { + fOpenWriteStream: func(ctx context.Context, name *common.BlobName) (WriteCloseCanceller, error) { return nil, errRet }, }} @@ -92,7 +92,7 @@ func TestDatastoreWriteFailure(t *testing.T) { closeCalled := false cancelCalled := false ds := &datastore{s: &mockStore{ - fOpenWriteStream: func(ctx context.Context, name common.BlobName) (WriteCloseCanceller, error) { + fOpenWriteStream: func(ctx context.Context, name *common.BlobName) (WriteCloseCanceller, error) { return &mockWriteCloseCanceller{ fWrite: func(b []byte) (int, error) { require.False(t, closeCalled) @@ -111,7 +111,7 @@ func TestDatastoreWriteFailure(t *testing.T) { }, }, nil }, - fOpenReadStream: func(ctx context.Context, name common.BlobName) (io.ReadCloser, error) { + fOpenReadStream: func(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { return nil, ErrNotFound }, }} diff --git a/pkg/datastore/interface.go b/pkg/datastore/interface.go index 3f24a68..06a7e61 100644 --- a/pkg/datastore/interface.go +++ b/pkg/datastore/interface.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -65,20 +65,20 @@ type DS interface { // If a non-nil error is returned, the writer will be nil. Otherwise it // is necessary to call the `Close` on the returned reader once done // with the reader. - Open(ctx context.Context, name common.BlobName) (io.ReadCloser, error) + Open(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) // Update retrieves an update for given blob. The data is read from given // reader until it returns either EOF, ending successful save, or any other // error which will cancel the save - in such case this error will be // returned from this function. If the data does not pass validation, // ErrInvalidData will be returned. - Update(ctx context.Context, name common.BlobName, r io.Reader) error + Update(ctx context.Context, name *common.BlobName, r io.Reader) error // Exists does check whether blob of given name exists in the datastore. // Partially written blobs are equal to non-existing ones. Boolean value // returned indicates whether the blob exists or not, non-nil error indicates // that there was an error while trying to check blob's existence. - Exists(ctx context.Context, name common.BlobName) (bool, error) + Exists(ctx context.Context, name *common.BlobName) (bool, error) // Delete tries to remove blob with given name from the datastore. // If blob does not exist (which includes partially written blobs) @@ -87,5 +87,5 @@ type DS interface { // read the blob data. After the `Delete` call succeeds, trying to read // the blob with the `Open` should end up with an ErrNotFound error // until the blob is updated again with a successful `Update` call. - Delete(ctx context.Context, name common.BlobName) error + Delete(ctx context.Context, name *common.BlobName) error } diff --git a/pkg/datastore/multi_source.go b/pkg/datastore/multi_source.go index a47cabb..e2084b7 100644 --- a/pkg/datastore/multi_source.go +++ b/pkg/datastore/multi_source.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -75,25 +75,25 @@ func (m *multiSourceDatastore) Address() string { return "multi-source://" } -func (m *multiSourceDatastore) Open(ctx context.Context, name common.BlobName) (io.ReadCloser, error) { +func (m *multiSourceDatastore) Open(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { m.fetch(ctx, name) return m.main.Open(ctx, name) } -func (m *multiSourceDatastore) Update(ctx context.Context, name common.BlobName, r io.Reader) error { +func (m *multiSourceDatastore) Update(ctx context.Context, name *common.BlobName, r io.Reader) error { return m.main.Update(ctx, name, r) } -func (m *multiSourceDatastore) Exists(ctx context.Context, name common.BlobName) (bool, error) { +func (m *multiSourceDatastore) Exists(ctx context.Context, name *common.BlobName) (bool, error) { m.fetch(ctx, name) return m.main.Exists(ctx, name) } -func (m *multiSourceDatastore) Delete(ctx context.Context, name common.BlobName) error { +func (m *multiSourceDatastore) Delete(ctx context.Context, name *common.BlobName) error { return m.main.Delete(ctx, name) } -func (m *multiSourceDatastore) fetch(ctx context.Context, name common.BlobName) { +func (m *multiSourceDatastore) fetch(ctx context.Context, name *common.BlobName) { // TODO: // if not found locally, go over all additional sources and check if exists, // for dynamic content, perform merge operation if found in more than one, diff --git a/pkg/datastore/multi_source_test.go b/pkg/datastore/multi_source_test.go index a6faa7b..a06badb 100644 --- a/pkg/datastore/multi_source_test.go +++ b/pkg/datastore/multi_source_test.go @@ -31,7 +31,7 @@ import ( func TestMultiSourceDatastore(t *testing.T) { - addBlob := func(ds DS, c string) common.BlobName { + addBlob := func(ds DS, c string) *common.BlobName { hash := sha256.Sum256([]byte(c)) name, err := common.BlobNameFromHashAndType(hash[:], blobtypes.Static) require.NoError(t, err) @@ -40,7 +40,7 @@ func TestMultiSourceDatastore(t *testing.T) { return name } - fetchBlob := func(ds DS, n common.BlobName) string { + fetchBlob := func(ds DS, n *common.BlobName) string { rc, err := ds.Open(context.Background(), n) require.NoError(t, err) @@ -53,7 +53,7 @@ func TestMultiSourceDatastore(t *testing.T) { return string(data) } - ensureNotFound := func(ds DS, n common.BlobName) { + ensureNotFound := func(ds DS, n *common.BlobName) { _, err := ds.Open(context.Background(), n) require.ErrorIs(t, err, ErrNotFound) } diff --git a/pkg/datastore/storage.go b/pkg/datastore/storage.go index f4c6545..d9d3bfa 100644 --- a/pkg/datastore/storage.go +++ b/pkg/datastore/storage.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -31,8 +31,8 @@ type WriteCloseCanceller interface { type storage interface { kind() string address() string - openReadStream(ctx context.Context, name common.BlobName) (io.ReadCloser, error) - openWriteStream(ctx context.Context, name common.BlobName) (WriteCloseCanceller, error) - exists(ctx context.Context, name common.BlobName) (bool, error) - delete(ctx context.Context, name common.BlobName) error + openReadStream(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) + openWriteStream(ctx context.Context, name *common.BlobName) (WriteCloseCanceller, error) + exists(ctx context.Context, name *common.BlobName) (bool, error) + delete(ctx context.Context, name *common.BlobName) error } diff --git a/pkg/datastore/storage_filesystem.go b/pkg/datastore/storage_filesystem.go index 4bea382..7472584 100644 --- a/pkg/datastore/storage_filesystem.go +++ b/pkg/datastore/storage_filesystem.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -52,7 +52,7 @@ func (fs *fileSystem) address() string { return filePrefix + fs.path } -func (fs *fileSystem) openReadStream(ctx context.Context, name common.BlobName) (io.ReadCloser, error) { +func (fs *fileSystem) openReadStream(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { rc, err := os.Open(fs.getFileName(name, fsSuffixCurrent)) if os.IsNotExist(err) { return nil, ErrNotFound @@ -60,7 +60,7 @@ func (fs *fileSystem) openReadStream(ctx context.Context, name common.BlobName) return rc, err } -func (fs *fileSystem) createTemporaryWriteStream(name common.BlobName) (*os.File, error) { +func (fs *fileSystem) createTemporaryWriteStream(name *common.BlobName) (*os.File, error) { tempName := fs.getFileName(name, fsSuffixUpload) // Ensure dir exists @@ -121,7 +121,7 @@ func (w *fileSystemWriteCloser) Close() error { return nil } -func (fs *fileSystem) openWriteStream(ctx context.Context, name common.BlobName) (WriteCloseCanceller, error) { +func (fs *fileSystem) openWriteStream(ctx context.Context, name *common.BlobName) (WriteCloseCanceller, error) { fl, err := fs.createTemporaryWriteStream(name) if err != nil { @@ -134,7 +134,7 @@ func (fs *fileSystem) openWriteStream(ctx context.Context, name common.BlobName) }, nil } -func (fs *fileSystem) exists(ctx context.Context, name common.BlobName) (bool, error) { +func (fs *fileSystem) exists(ctx context.Context, name *common.BlobName) (bool, error) { _, err := os.Stat(fs.getFileName(name, fsSuffixCurrent)) if os.IsNotExist(err) { return false, nil @@ -145,7 +145,7 @@ func (fs *fileSystem) exists(ctx context.Context, name common.BlobName) (bool, e return true, nil } -func (fs *fileSystem) delete(ctx context.Context, name common.BlobName) error { +func (fs *fileSystem) delete(ctx context.Context, name *common.BlobName) error { err := os.Remove(fs.getFileName(name, fsSuffixCurrent)) if os.IsNotExist(err) { return ErrNotFound @@ -153,7 +153,7 @@ func (fs *fileSystem) delete(ctx context.Context, name common.BlobName) error { return err } -func (fs *fileSystem) getFileName(name common.BlobName, suffix string) string { +func (fs *fileSystem) getFileName(name *common.BlobName, suffix string) string { fNameParts := []string{fs.path} nameStr := name.String() diff --git a/pkg/datastore/storage_memory.go b/pkg/datastore/storage_memory.go index 90cabc7..3abf51e 100644 --- a/pkg/datastore/storage_memory.go +++ b/pkg/datastore/storage_memory.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -54,7 +54,7 @@ func (m *memory) address() string { return memoryPrefix } -func (m *memory) openReadStream(ctx context.Context, name common.BlobName) (io.ReadCloser, error) { +func (m *memory) openReadStream(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { m.rw.RLock() defer m.rw.RUnlock() @@ -92,7 +92,7 @@ func (w *memoryWriteCloser) Close() error { return nil } -func (m *memory) openWriteStream(ctx context.Context, name common.BlobName) (WriteCloseCanceller, error) { +func (m *memory) openWriteStream(ctx context.Context, name *common.BlobName) (WriteCloseCanceller, error) { m.rw.Lock() defer m.rw.Unlock() @@ -111,7 +111,7 @@ func (m *memory) openWriteStream(ctx context.Context, name common.BlobName) (Wri }, nil } -func (m *memory) exists(ctx context.Context, n common.BlobName) (bool, error) { +func (m *memory) exists(ctx context.Context, n *common.BlobName) (bool, error) { m.rw.RLock() defer m.rw.RUnlock() @@ -122,7 +122,7 @@ func (m *memory) exists(ctx context.Context, n common.BlobName) (bool, error) { return true, nil } -func (m *memory) delete(ctx context.Context, n common.BlobName) error { +func (m *memory) delete(ctx context.Context, n *common.BlobName) error { m.rw.Lock() defer m.rw.Unlock() diff --git a/pkg/datastore/storage_raw_filesystem.go b/pkg/datastore/storage_raw_filesystem.go index 7575403..a7f8253 100644 --- a/pkg/datastore/storage_raw_filesystem.go +++ b/pkg/datastore/storage_raw_filesystem.go @@ -50,7 +50,7 @@ func (fs *rawFileSystem) address() string { return rawFilePrefix + fs.path } -func (fs *rawFileSystem) openReadStream(ctx context.Context, name common.BlobName) (io.ReadCloser, error) { +func (fs *rawFileSystem) openReadStream(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { rc, err := os.Open(filepath.Join(fs.path, name.String())) if os.IsNotExist(err) { return nil, ErrNotFound @@ -81,7 +81,7 @@ func (w *rawFilesystemWriter) Cancel() { os.Remove(w.file.Name()) } -func (fs *rawFileSystem) openWriteStream(ctx context.Context, name common.BlobName) (WriteCloseCanceller, error) { +func (fs *rawFileSystem) openWriteStream(ctx context.Context, name *common.BlobName) (WriteCloseCanceller, error) { tempNum := atomic.AddUint64(&fs.tempFileNum, 1) tempFileName := filepath.Join(fs.path, fmt.Sprintf("tempfile_%d", tempNum)) @@ -97,7 +97,7 @@ func (fs *rawFileSystem) openWriteStream(ctx context.Context, name common.BlobNa }, nil } -func (fs *rawFileSystem) exists(ctx context.Context, name common.BlobName) (bool, error) { +func (fs *rawFileSystem) exists(ctx context.Context, name *common.BlobName) (bool, error) { _, err := os.Stat(filepath.Join(fs.path, name.String())) if os.IsNotExist(err) { return false, nil @@ -108,7 +108,7 @@ func (fs *rawFileSystem) exists(ctx context.Context, name common.BlobName) (bool return true, nil } -func (fs *rawFileSystem) delete(ctx context.Context, name common.BlobName) error { +func (fs *rawFileSystem) delete(ctx context.Context, name *common.BlobName) error { err := os.Remove(filepath.Join(fs.path, name.String())) if os.IsNotExist(err) { return ErrNotFound diff --git a/pkg/datastore/storage_test.go b/pkg/datastore/storage_test.go index 1a3f10b..aeed0ef 100644 --- a/pkg/datastore/storage_test.go +++ b/pkg/datastore/storage_test.go @@ -121,7 +121,7 @@ func TestStorageSaveOpenCancelSuccess(t *testing.T) { func TestStorageDelete(t *testing.T) { for _, st := range allTestStorages(t) { t.Run(st.kind(), func(t *testing.T) { - blobNames := []common.BlobName{} + blobNames := []*common.BlobName{} blobDatas := [][]byte{} t.Run("generate test data", func(t *testing.T) { diff --git a/pkg/datastore/utils_autogen_for_test.go b/pkg/datastore/utils_autogen_for_test.go index 2e928e1..10a3c2b 100644 --- a/pkg/datastore/utils_autogen_for_test.go +++ b/pkg/datastore/utils_autogen_for_test.go @@ -31,7 +31,7 @@ import ( ) var testBlobs = []struct { - name common.BlobName + name *common.BlobName data []byte expected []byte }{ @@ -69,7 +69,7 @@ var testBlobs = []struct { } var dynamicLinkPropagationData = []struct { - name common.BlobName + name *common.BlobName data []byte expected []byte }{ @@ -93,7 +93,7 @@ var dynamicLinkPropagationData = []struct { func TestDatasetGeneration(t *testing.T) { t.SkipNow() - dumpBlob := func(name common.BlobName, content []byte, expected []byte) { + dumpBlob := func(name *common.BlobName, content []byte, expected []byte) { fmt.Printf(""+ " {\n"+ " golang.Must(common.BlobNameFromString(\"%s\")),\n"+ diff --git a/pkg/datastore/utils_for_test.go b/pkg/datastore/utils_for_test.go index a7926e4..afc8f50 100644 --- a/pkg/datastore/utils_for_test.go +++ b/pkg/datastore/utils_for_test.go @@ -25,7 +25,7 @@ import ( "github.com/cinode/go/pkg/common" ) -var emptyBlobNameStatic = func() common.BlobName { +var emptyBlobNameStatic = func() *common.BlobName { bn, err := common.BlobNameFromHashAndType(sha256.New().Sum(nil), blobtypes.Static) if err != nil { panic(err) @@ -33,7 +33,7 @@ var emptyBlobNameStatic = func() common.BlobName { return bn }() -var emptyBlobNameDynamicLink = func() common.BlobName { +var emptyBlobNameDynamicLink = func() *common.BlobName { bn, err := common.BlobNameFromHashAndType(sha256.New().Sum(nil), blobtypes.DynamicLink) if err != nil { panic(err) @@ -41,7 +41,7 @@ var emptyBlobNameDynamicLink = func() common.BlobName { return bn }() -var emptyBlobNamesOfAllTypes = []common.BlobName{ +var emptyBlobNamesOfAllTypes = []*common.BlobName{ emptyBlobNameStatic, emptyBlobNameDynamicLink, } diff --git a/pkg/datastore/webconnector.go b/pkg/datastore/webconnector.go index f015f39..3017e54 100644 --- a/pkg/datastore/webconnector.go +++ b/pkg/datastore/webconnector.go @@ -83,7 +83,7 @@ func (w *webConnector) Address() string { return w.baseURL } -func (w *webConnector) Open(ctx context.Context, name common.BlobName) (io.ReadCloser, error) { +func (w *webConnector) Open(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { switch name.Type() { case blobtypes.Static: return w.openStatic(ctx, name) @@ -94,7 +94,7 @@ func (w *webConnector) Open(ctx context.Context, name common.BlobName) (io.ReadC } } -func (w *webConnector) openStatic(ctx context.Context, name common.BlobName) (io.ReadCloser, error) { +func (w *webConnector) openStatic(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { req, err := http.NewRequestWithContext( ctx, http.MethodGet, @@ -125,7 +125,7 @@ func (w *webConnector) openStatic(ctx context.Context, name common.BlobName) (io }, nil } -func (w *webConnector) openDynamicLink(ctx context.Context, name common.BlobName) (io.ReadCloser, error) { +func (w *webConnector) openDynamicLink(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { req, err := http.NewRequestWithContext( ctx, http.MethodGet, @@ -156,7 +156,7 @@ func (w *webConnector) openDynamicLink(ctx context.Context, name common.BlobName return io.NopCloser(bytes.NewReader(buff.Bytes())), nil } -func (w *webConnector) Update(ctx context.Context, name common.BlobName, r io.Reader) error { +func (w *webConnector) Update(ctx context.Context, name *common.BlobName, r io.Reader) error { req, err := http.NewRequestWithContext( ctx, http.MethodPut, @@ -178,7 +178,7 @@ func (w *webConnector) Update(ctx context.Context, name common.BlobName, r io.Re return w.errCheck(res) } -func (w *webConnector) Exists(ctx context.Context, name common.BlobName) (bool, error) { +func (w *webConnector) Exists(ctx context.Context, name *common.BlobName) (bool, error) { req, err := http.NewRequestWithContext( ctx, http.MethodHead, @@ -205,7 +205,7 @@ func (w *webConnector) Exists(ctx context.Context, name common.BlobName) (bool, return false, err } -func (w *webConnector) Delete(ctx context.Context, name common.BlobName) error { +func (w *webConnector) Delete(ctx context.Context, name *common.BlobName) error { req, err := http.NewRequestWithContext( ctx, http.MethodDelete, diff --git a/pkg/datastore/webinterface.go b/pkg/datastore/webinterface.go index a4f0eac..5d756fb 100644 --- a/pkg/datastore/webinterface.go +++ b/pkg/datastore/webinterface.go @@ -76,21 +76,21 @@ func (i *webInterface) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } -func (i *webInterface) getName(w http.ResponseWriter, r *http.Request) (common.BlobName, error) { +func (i *webInterface) getName(w http.ResponseWriter, r *http.Request) (*common.BlobName, error) { // Don't allow url queries and require path to start with '/' if r.URL.Path[0] != '/' || r.URL.RawQuery != "" { - return common.BlobName{}, common.ErrInvalidBlobName + return nil, common.ErrInvalidBlobName } bn, err := common.BlobNameFromString(r.URL.Path[1:]) if err != nil { - return common.BlobName{}, err + return nil, err } return bn, nil } -func (i *webInterface) sendName(name common.BlobName, w http.ResponseWriter, r *http.Request) { +func (i *webInterface) sendName(name *common.BlobName, w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-type", "application/json") json.NewEncoder(w).Encode(&webNameResponse{ Name: name.String(), diff --git a/pkg/datastore/webinterface_test.go b/pkg/datastore/webinterface_test.go index e733108..8d2cd28 100644 --- a/pkg/datastore/webinterface_test.go +++ b/pkg/datastore/webinterface_test.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -97,7 +97,7 @@ func TestWebInterfaceDeleteQueryString(t *testing.T) { func TestWebIntefaceExistsFailure(t *testing.T) { server := httptest.NewServer(WebInterface(&datastore{ s: &mockStore{ - fExists: func(ctx context.Context, name common.BlobName) (bool, error) { return false, errors.New("fail") }, + fExists: func(ctx context.Context, name *common.BlobName) (bool, error) { return false, errors.New("fail") }, }, })) defer server.Close() diff --git a/pkg/internal/blobtypes/dynamiclink/public.go b/pkg/internal/blobtypes/dynamiclink/public.go index f5f3f1a..ded9c80 100644 --- a/pkg/internal/blobtypes/dynamiclink/public.go +++ b/pkg/internal/blobtypes/dynamiclink/public.go @@ -61,7 +61,7 @@ type Public struct { nonce uint64 } -func (d *Public) BlobName() common.BlobName { +func (d *Public) BlobName() *common.BlobName { hasher := sha256.New() storeByte(hasher, reservedByteValue) @@ -80,7 +80,7 @@ type PublicReader struct { Public contentVersion uint64 signature []byte - iv common.BlobIV + iv *common.BlobIV r io.Reader } @@ -88,7 +88,7 @@ type PublicReader struct { // // Invalid links are rejected - i.e. if there's any error while reading the data // or when the validation of the link fails for whatever reason -func FromPublicData(name common.BlobName, r io.Reader) (*PublicReader, error) { +func FromPublicData(name *common.BlobName, r io.Reader) (*PublicReader, error) { dl := PublicReader{ Public: Public{ publicKey: make([]byte, ed25519.PublicKeySize), @@ -245,7 +245,7 @@ func (d *PublicReader) ivGeneratorPrefilled() cipherfactory.IVGenerator { return ivGenerator } -func (d *PublicReader) validateKeyInLinkData(key common.BlobKey, r io.Reader) error { +func (d *PublicReader) validateKeyInLinkData(key *common.BlobKey, r io.Reader) error { // At the beginning of the data there's the key validation block, // that block contains a proof that the encryption key was deterministically derived // from the blob name (thus preventing weak key attack) @@ -277,7 +277,7 @@ func (d *PublicReader) validateKeyInLinkData(key common.BlobKey, r io.Reader) er return nil } -func (d *PublicReader) GetLinkDataReader(key common.BlobKey) (io.Reader, error) { +func (d *PublicReader) GetLinkDataReader(key *common.BlobKey) (io.Reader, error) { r, err := cipherfactory.StreamCipherReader(key, d.iv, d.GetEncryptedLinkReader()) if err != nil { diff --git a/pkg/internal/blobtypes/dynamiclink/public_test.go b/pkg/internal/blobtypes/dynamiclink/public_test.go index b02d36d..5c030eb 100644 --- a/pkg/internal/blobtypes/dynamiclink/public_test.go +++ b/pkg/internal/blobtypes/dynamiclink/public_test.go @@ -37,7 +37,7 @@ func TestFromPublicData(t *testing.T) { t.Run("Ensure we don't crash on truncated data", func(t *testing.T) { for i := 0; i < 1000; i++ { data := make([]byte, i) - dl, err := FromPublicData(common.BlobName{}, bytes.NewReader(data)) + dl, err := FromPublicData(&common.BlobName{}, bytes.NewReader(data)) require.ErrorIs(t, err, ErrInvalidDynamicLinkData) require.Nil(t, dl) } @@ -45,7 +45,7 @@ func TestFromPublicData(t *testing.T) { t.Run("Do not accept the link if reserved byte is not zero", func(t *testing.T) { data := []byte{0xFF, 0, 0, 0} - dl, err := FromPublicData(common.BlobName{}, bytes.NewReader(data)) + dl, err := FromPublicData(&common.BlobName{}, bytes.NewReader(data)) require.ErrorIs(t, err, ErrInvalidDynamicLinkData) require.ErrorIs(t, err, ErrInvalidDynamicLinkDataReservedByte) require.Nil(t, dl) @@ -324,7 +324,7 @@ func TestPublicReaderGetLinkDataReader(t *testing.T) { pr, _, err := link.UpdateLinkData(bytes.NewReader([]byte("Hello world")), 0) require.NoError(t, err) - _, err = pr.GetLinkDataReader(common.BlobKey{}) + _, err = pr.GetLinkDataReader(&common.BlobKey{}) require.ErrorIs(t, err, cipherfactory.ErrInvalidEncryptionConfigKeyType) }) } diff --git a/pkg/internal/blobtypes/dynamiclink/publisher.go b/pkg/internal/blobtypes/dynamiclink/publisher.go index 66ad207..d306598 100644 --- a/pkg/internal/blobtypes/dynamiclink/publisher.go +++ b/pkg/internal/blobtypes/dynamiclink/publisher.go @@ -107,7 +107,7 @@ func (dl *Publisher) AuthInfo() []byte { return ret[:] } -func (dl *Publisher) calculateEncryptionKey() (common.BlobKey, []byte) { +func (dl *Publisher) calculateEncryptionKey() (*common.BlobKey, []byte) { dataSeed := append( []byte{signatureForEncryptionKeyGeneration}, dl.BlobName().Bytes()..., @@ -123,12 +123,12 @@ func (dl *Publisher) calculateEncryptionKey() (common.BlobKey, []byte) { return key, signature } -func (dl *Publisher) EncryptionKey() common.BlobKey { +func (dl *Publisher) EncryptionKey() *common.BlobKey { key, _ := dl.calculateEncryptionKey() return key } -func (dl *Publisher) UpdateLinkData(r io.Reader, version uint64) (*PublicReader, common.BlobKey, error) { +func (dl *Publisher) UpdateLinkData(r io.Reader, version uint64) (*PublicReader, *common.BlobKey, error) { encryptionKey, kvb := dl.calculateEncryptionKey() // key validation block precedes the link data @@ -138,7 +138,7 @@ func (dl *Publisher) UpdateLinkData(r io.Reader, version uint64) (*PublicReader, _, err := io.Copy(unencryptedLinkBuff, r) if err != nil { - return nil, common.BlobKey{}, err + return nil, nil, err } unencryptedLink := unencryptedLinkBuff.Bytes() @@ -156,12 +156,12 @@ func (dl *Publisher) UpdateLinkData(r io.Reader, version uint64) (*PublicReader, encryptedLinkBuff := bytes.NewBuffer(nil) w, err := cipherfactory.StreamCipherWriter(encryptionKey, pr.iv, encryptedLinkBuff) if err != nil { - return nil, common.BlobKey{}, err + return nil, nil, err } _, err = w.Write(unencryptedLink) if err != nil { - return nil, common.BlobKey{}, err + return nil, nil, err } signatureHasher := pr.toSignDataHasherPrefilled() diff --git a/pkg/internal/blobtypes/dynamiclink/publisher_test.go b/pkg/internal/blobtypes/dynamiclink/publisher_test.go index 47e9687..2e84624 100644 --- a/pkg/internal/blobtypes/dynamiclink/publisher_test.go +++ b/pkg/internal/blobtypes/dynamiclink/publisher_test.go @@ -24,7 +24,6 @@ import ( "testing" "testing/iotest" - "github.com/cinode/go/pkg/common" "github.com/stretchr/testify/require" ) @@ -80,7 +79,7 @@ func TestFromAuthInfo(t *testing.T) { }) } -func TestRenonc(t *testing.T) { +func TestReNonce(t *testing.T) { dl1, err := Create(rand.Reader) require.NoError(t, err) @@ -132,6 +131,6 @@ func TestPublisherUpdateLinkData(t *testing.T) { pr2, key2, err := dl.UpdateLinkData(iotest.ErrReader(injectedErr), 3) require.ErrorIs(t, err, injectedErr) require.Nil(t, pr2) - require.Equal(t, common.BlobKey{}, key2) + require.Nil(t, key2) }) } diff --git a/pkg/internal/utilities/cipherfactory/cipher_factory.go b/pkg/internal/utilities/cipherfactory/cipher_factory.go index 171c2de..6d8f3c0 100644 --- a/pkg/internal/utilities/cipherfactory/cipher_factory.go +++ b/pkg/internal/utilities/cipherfactory/cipher_factory.go @@ -40,7 +40,7 @@ const ( reservedByteForKeyType byte = 0 ) -func StreamCipherReader(key common.BlobKey, iv common.BlobIV, r io.Reader) (io.Reader, error) { +func StreamCipherReader(key *common.BlobKey, iv *common.BlobIV, r io.Reader) (io.Reader, error) { stream, err := _cipherForKeyIV(key, iv) if err != nil { return nil, err @@ -48,7 +48,7 @@ func StreamCipherReader(key common.BlobKey, iv common.BlobIV, r io.Reader) (io.R return &cipher.StreamReader{S: stream, R: r}, nil } -func StreamCipherWriter(key common.BlobKey, iv common.BlobIV, w io.Writer) (io.Writer, error) { +func StreamCipherWriter(key *common.BlobKey, iv *common.BlobIV, w io.Writer) (io.Writer, error) { stream, err := _cipherForKeyIV(key, iv) if err != nil { return nil, err @@ -56,7 +56,7 @@ func StreamCipherWriter(key common.BlobKey, iv common.BlobIV, w io.Writer) (io.W return cipher.StreamWriter{S: stream, W: w}, nil } -func _cipherForKeyIV(key common.BlobKey, iv common.BlobIV) (cipher.Stream, error) { +func _cipherForKeyIV(key *common.BlobKey, iv *common.BlobIV) (cipher.Stream, error) { keyBytes := key.Bytes() if len(keyBytes) == 0 || keyBytes[0] != reservedByteForKeyType { return nil, ErrInvalidEncryptionConfigKeyType diff --git a/pkg/internal/utilities/cipherfactory/generator.go b/pkg/internal/utilities/cipherfactory/generator.go index 9f9c3da..06388b8 100644 --- a/pkg/internal/utilities/cipherfactory/generator.go +++ b/pkg/internal/utilities/cipherfactory/generator.go @@ -33,7 +33,7 @@ const ( type KeyGenerator interface { io.Writer - Generate() common.BlobKey + Generate() *common.BlobKey } type keyGenerator struct { @@ -42,7 +42,7 @@ type keyGenerator struct { func (g keyGenerator) Write(b []byte) (int, error) { return g.h.Write(b) } -func (g keyGenerator) Generate() common.BlobKey { +func (g keyGenerator) Generate() *common.BlobKey { return common.BlobKeyFromBytes(append( []byte{reservedByteForKeyType}, g.h.Sum(nil)[:chacha20.KeySize]..., @@ -51,7 +51,7 @@ func (g keyGenerator) Generate() common.BlobKey { type IVGenerator interface { io.Writer - Generate() common.BlobIV + Generate() *common.BlobIV } type ivGenerator struct { @@ -60,7 +60,7 @@ type ivGenerator struct { func (g ivGenerator) Write(b []byte) (int, error) { return g.h.Write(b) } -func (g ivGenerator) Generate() common.BlobIV { +func (g ivGenerator) Generate() *common.BlobIV { return common.BlobIVFromBytes(g.h.Sum(nil)[:chacha20.NonceSizeX]) } @@ -68,7 +68,6 @@ func NewKeyGenerator(t common.BlobType) KeyGenerator { h := sha256.New() h.Write([]byte{preambleHashKey, reservedByteForKeyType, t.IDByte()}) return keyGenerator{h: h} - } func NewIVGenerator(t common.BlobType) IVGenerator { @@ -77,12 +76,12 @@ func NewIVGenerator(t common.BlobType) IVGenerator { return ivGenerator{h: h} } -var defaultXChaCha20IV = func() common.BlobIV { +var defaultXChaCha20IV = func() *common.BlobIV { h := sha256.New() h.Write([]byte{preambleHashDefaultIV, reservedByteForKeyType}) return common.BlobIVFromBytes(h.Sum(nil)[:chacha20.NonceSizeX]) }() -func DefaultIV(k common.BlobKey) common.BlobIV { +func DefaultIV(k *common.BlobKey) *common.BlobIV { return defaultXChaCha20IV } diff --git a/testvectors/testblobs/base.go b/testvectors/testblobs/base.go index 78da468..ee1c107 100644 --- a/testvectors/testblobs/base.go +++ b/testvectors/testblobs/base.go @@ -25,13 +25,12 @@ import ( "github.com/cinode/go/pkg/cinodefs" "github.com/cinode/go/pkg/common" - "github.com/jbenet/go-base58" ) type TestBlob struct { UpdateDataset []byte - BlobName common.BlobName - EncryptionKey common.BlobKey + BlobName *common.BlobName + EncryptionKey *common.BlobKey DecryptedDataset []byte } @@ -40,7 +39,7 @@ func (s *TestBlob) Put(baseUrl string) error { } func (s *TestBlob) PutWithAuth(baseUrl, username, password string) error { - finalUrl, err := url.JoinPath(baseUrl, base58.Encode(s.BlobName)) + finalUrl, err := url.JoinPath(baseUrl, s.BlobName.String()) if err != nil { return err } @@ -76,7 +75,7 @@ func (s *TestBlob) PutWithAuth(baseUrl, username, password string) error { } func (s *TestBlob) Get(baseUrl string) ([]byte, error) { - finalUrl, err := url.JoinPath(baseUrl, base58.Encode(s.BlobName)) + finalUrl, err := url.JoinPath(baseUrl, s.BlobName.String()) if err != nil { return nil, err } From da64207f5ecfd8e1961d21f0adc326f576b225a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Sun, 29 Oct 2023 13:32:12 +0100 Subject: [PATCH 16/29] Extract AuthInfo to a separate type --- pkg/blenc/datastore.go | 4 +-- pkg/blenc/datastore_dynamic_link.go | 4 +-- pkg/blenc/datastore_static.go | 4 +-- pkg/blenc/interface.go | 4 +-- pkg/blenc/interface_test.go | 5 ++-- pkg/cinodefs/cinodefs.go | 9 +++--- pkg/cinodefs/cinodefs_options.go | 2 +- pkg/cinodefs/context.go | 6 ++-- pkg/cinodefs/node_link.go | 4 +-- pkg/cinodefs/writerinfo.go | 4 +-- pkg/common/auth_info.go | 29 +++++++++++++++++++ .../blobtypes/dynamiclink/publisher.go | 13 +++++---- .../blobtypes/dynamiclink/publisher_test.go | 7 +++-- 13 files changed, 65 insertions(+), 30 deletions(-) create mode 100644 pkg/common/auth_info.go diff --git a/pkg/blenc/datastore.go b/pkg/blenc/datastore.go index d23ea4b..07fe94b 100644 --- a/pkg/blenc/datastore.go +++ b/pkg/blenc/datastore.go @@ -67,7 +67,7 @@ func (be *beDatastore) Create( ) ( *common.BlobName, *common.BlobKey, - AuthInfo, + *common.AuthInfo, error, ) { switch blobType { @@ -79,7 +79,7 @@ func (be *beDatastore) Create( return nil, nil, nil, blobtypes.ErrUnknownBlobType } -func (be *beDatastore) Update(ctx context.Context, name *common.BlobName, authInfo AuthInfo, key *common.BlobKey, r io.Reader) error { +func (be *beDatastore) Update(ctx context.Context, name *common.BlobName, authInfo *common.AuthInfo, key *common.BlobKey, r io.Reader) error { switch name.Type() { case blobtypes.Static: return be.updateStatic(ctx, name, authInfo, key, r) diff --git a/pkg/blenc/datastore_dynamic_link.go b/pkg/blenc/datastore_dynamic_link.go index 3d754c0..fdb670a 100644 --- a/pkg/blenc/datastore_dynamic_link.go +++ b/pkg/blenc/datastore_dynamic_link.go @@ -77,7 +77,7 @@ func (be *beDatastore) createDynamicLink( ) ( *common.BlobName, *common.BlobKey, - AuthInfo, + *common.AuthInfo, error, ) { version := be.generateVersion() @@ -108,7 +108,7 @@ func (be *beDatastore) createDynamicLink( func (be *beDatastore) updateDynamicLink( ctx context.Context, name *common.BlobName, - authInfo AuthInfo, + authInfo *common.AuthInfo, key *common.BlobKey, r io.Reader, ) error { diff --git a/pkg/blenc/datastore_static.go b/pkg/blenc/datastore_static.go index 71e88df..1dc28c4 100644 --- a/pkg/blenc/datastore_static.go +++ b/pkg/blenc/datastore_static.go @@ -69,7 +69,7 @@ func (be *beDatastore) createStatic( ) ( *common.BlobName, *common.BlobKey, - AuthInfo, + *common.AuthInfo, error, ) { tempWriteBufferPlain, err := be.newSecureFifo() @@ -141,7 +141,7 @@ func (be *beDatastore) createStatic( func (be *beDatastore) updateStatic( ctx context.Context, name *common.BlobName, - authInfo AuthInfo, + authInfo *common.AuthInfo, key *common.BlobKey, r io.Reader, ) error { diff --git a/pkg/blenc/interface.go b/pkg/blenc/interface.go index 4124d1d..b793ee2 100644 --- a/pkg/blenc/interface.go +++ b/pkg/blenc/interface.go @@ -43,13 +43,13 @@ type BE interface { // Create completely new blob with given dataset, as a result, the blob name and optional // AuthInfo that allows blob's update is returned - Create(ctx context.Context, blobType common.BlobType, r io.Reader) (*common.BlobName, *common.BlobKey, AuthInfo, error) + Create(ctx context.Context, blobType common.BlobType, r io.Reader) (*common.BlobName, *common.BlobKey, *common.AuthInfo, error) // Update updates given blob type with new data, // The update must happen within a single blob name (i.e. it can not end up with blob with different name) // and may not be available for certain blob types such as static blobs. // A valid auth info is necessary to ensure a correct new content can be created - Update(ctx context.Context, name *common.BlobName, ai AuthInfo, key *common.BlobKey, r io.Reader) error + Update(ctx context.Context, name *common.BlobName, ai *common.AuthInfo, key *common.BlobKey, r io.Reader) error // Exists does check whether blob of given name exists. It forwards the call // to underlying datastore. diff --git a/pkg/blenc/interface_test.go b/pkg/blenc/interface_test.go index ff2f472..2d6b896 100644 --- a/pkg/blenc/interface_test.go +++ b/pkg/blenc/interface_test.go @@ -225,7 +225,8 @@ func (s *BlencTestSuite) TestDynamicLinkSuccessPath() { }) s.Run("must fail to update if auth info is invalid", func() { - err := s.be.Update(context.Background(), bn, ai2[1:], key2, bytes.NewReader(nil)) + brokenAI2 := common.AuthInfoFromBytes(ai2.Bytes()[1:]) + err := s.be.Update(context.Background(), bn, brokenAI2, key2, bytes.NewReader(nil)) s.Require().ErrorIs(err, dynamiclink.ErrInvalidDynamicLinkAuthInfo) }) @@ -275,7 +276,7 @@ func (s *BlencTestSuite) TestInvalidBlobTypes() { err = s.be.Update( context.Background(), invalidBlobName, - AuthInfo{}, + nil, nil, bytes.NewReader(nil), ) diff --git a/pkg/cinodefs/cinodefs.go b/pkg/cinodefs/cinodefs.go index 5ed5b53..9501adb 100644 --- a/pkg/cinodefs/cinodefs.go +++ b/pkg/cinodefs/cinodefs.go @@ -28,6 +28,7 @@ import ( "github.com/cinode/go/pkg/blenc" "github.com/cinode/go/pkg/blobtypes" + "github.com/cinode/go/pkg/common" "github.com/cinode/go/pkg/internal/blobtypes/dynamiclink" ) @@ -137,8 +138,8 @@ func New( timeFunc: time.Now, randSource: rand.Reader, c: graphContext{ - be: be, - writerInfos: map[string][]byte{}, + be: be, + authInfos: map[string]*common.AuthInfo{}, }, } @@ -341,7 +342,7 @@ func (fs *cinodeFS) GenerateNewDynamicLinkEntrypoint() (*Entrypoint, error) { bn := link.BlobName() key := link.EncryptionKey() - fs.c.writerInfos[bn.String()] = link.AuthInfo() + fs.c.authInfos[bn.String()] = link.AuthInfo() return EntrypointFromBlobNameAndKey(bn, key), nil } @@ -374,7 +375,7 @@ func (fs *cinodeFS) EntrypointWriterInfo(ctx context.Context, ep *Entrypoint) (* return nil, err } - authInfo, found := fs.c.writerInfos[bn.String()] + authInfo, found := fs.c.authInfos[bn.String()] if !found { return nil, ErrMissingWriterInfo } diff --git a/pkg/cinodefs/cinodefs_options.go b/pkg/cinodefs/cinodefs_options.go index 8056afb..9feefa8 100644 --- a/pkg/cinodefs/cinodefs_options.go +++ b/pkg/cinodefs/cinodefs_options.go @@ -75,7 +75,7 @@ func RootWriterInfo(wi *WriterInfo) Option { return optionFunc(func(ctx context.Context, fs *cinodeFS) error { fs.rootEP = &nodeUnloaded{ep: ep} - fs.c.writerInfos[bn.String()] = wi.wi.AuthInfo + fs.c.authInfos[bn.String()] = common.AuthInfoFromBytes(wi.wi.AuthInfo) return nil }) } diff --git a/pkg/cinodefs/context.go b/pkg/cinodefs/context.go index bfa1135..9e543e4 100644 --- a/pkg/cinodefs/context.go +++ b/pkg/cinodefs/context.go @@ -39,7 +39,7 @@ type graphContext struct { be blenc.BE // known writer info data - writerInfos map[string][]byte + authInfos map[string]*common.AuthInfo } // Get symmetric encryption key for given entrypoint. @@ -121,7 +121,7 @@ func (c *graphContext) createProtobufMessage( } if wi != nil { - c.writerInfos[bn.String()] = wi + c.authInfos[bn.String()] = wi } return &Entrypoint{ @@ -140,7 +140,7 @@ func (c *graphContext) updateProtobufMessage( ep *Entrypoint, msg proto.Message, ) error { - wi, found := c.writerInfos[ep.BlobName().String()] + wi, found := c.authInfos[ep.BlobName().String()] if !found { return ErrMissingWriterInfo } diff --git a/pkg/cinodefs/node_link.go b/pkg/cinodefs/node_link.go index 9a1e81e..ede746a 100644 --- a/pkg/cinodefs/node_link.go +++ b/pkg/cinodefs/node_link.go @@ -82,7 +82,7 @@ func (c *nodeLink) traverse( // crossing link border, whether sub-graph is writeable is determined // by availability of corresponding writer info - _, hasWriterInfo := gc.writerInfos[c.ep.bn.String()] + _, hasAuthInfo := gc.authInfos[c.ep.bn.String()] newTarget, targetState, err := c.target.traverse( ctx, @@ -90,7 +90,7 @@ func (c *nodeLink) traverse( path, pathPosition, linkDepth+1, - hasWriterInfo, + hasAuthInfo, opts, whenReached, ) diff --git a/pkg/cinodefs/writerinfo.go b/pkg/cinodefs/writerinfo.go index ca6bfaf..e9e562c 100644 --- a/pkg/cinodefs/writerinfo.go +++ b/pkg/cinodefs/writerinfo.go @@ -68,12 +68,12 @@ func WriterInfoFromBytes(b []byte) (*WriterInfo, error) { return &wi, nil } -func writerInfoFromBlobNameKeyAndAuthInfo(bn *common.BlobName, key *common.BlobKey, ai []byte) *WriterInfo { +func writerInfoFromBlobNameKeyAndAuthInfo(bn *common.BlobName, key *common.BlobKey, authInfo *common.AuthInfo) *WriterInfo { return &WriterInfo{ wi: protobuf.WriterInfo{ BlobName: bn.Bytes(), Key: key.Bytes(), - AuthInfo: ai, + AuthInfo: authInfo.Bytes(), }, } } diff --git a/pkg/common/auth_info.go b/pkg/common/auth_info.go new file mode 100644 index 0000000..f926d7c --- /dev/null +++ b/pkg/common/auth_info.go @@ -0,0 +1,29 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import "crypto/subtle" + +// AuthInfo is an opaque data that is necessary to perform update of an existing blob. +// +// Currently used only for dynamic links, auth info contains all the necessary information +// to update the content of the blob. The representation is specific to the blob type +type AuthInfo struct{ data []byte } + +func AuthInfoFromBytes(iv []byte) *AuthInfo { return &AuthInfo{data: copyBytes(iv)} } +func (a *AuthInfo) Bytes() []byte { return copyBytes(a.data) } +func (a *AuthInfo) Equal(a2 *AuthInfo) bool { return subtle.ConstantTimeCompare(a.data, a2.data) == 1 } diff --git a/pkg/internal/blobtypes/dynamiclink/publisher.go b/pkg/internal/blobtypes/dynamiclink/publisher.go index d306598..b296d7a 100644 --- a/pkg/internal/blobtypes/dynamiclink/publisher.go +++ b/pkg/internal/blobtypes/dynamiclink/publisher.go @@ -66,14 +66,15 @@ func Create(randSource io.Reader) (*Publisher, error) { }, nil } -func FromAuthInfo(authInfo []byte) (*Publisher, error) { - if len(authInfo) != 1+ed25519.SeedSize+8 || authInfo[0] != 0 { +func FromAuthInfo(authInfo *common.AuthInfo) (*Publisher, error) { + authInfoBytes := authInfo.Bytes() + if len(authInfoBytes) != 1+ed25519.SeedSize+8 || authInfoBytes[0] != 0 { return nil, ErrInvalidDynamicLinkAuthInfo } - privKey := ed25519.NewKeyFromSeed(authInfo[1 : 1+ed25519.SeedSize]) + privKey := ed25519.NewKeyFromSeed(authInfoBytes[1 : 1+ed25519.SeedSize]) pubKey := privKey.Public().(ed25519.PublicKey) - nonce := binary.BigEndian.Uint64(authInfo[1+ed25519.SeedSize:]) + nonce := binary.BigEndian.Uint64(authInfoBytes[1+ed25519.SeedSize:]) return &Publisher{ Public: Public{ @@ -99,12 +100,12 @@ func ReNonce(p *Publisher, randSource io.Reader) (*Publisher, error) { }, nil } -func (dl *Publisher) AuthInfo() []byte { +func (dl *Publisher) AuthInfo() *common.AuthInfo { var ret [1 + ed25519.SeedSize + 8]byte ret[0] = reservedByteValue copy(ret[1:], dl.privKey.Seed()) binary.BigEndian.PutUint64(ret[1+ed25519.SeedSize:], dl.nonce) - return ret[:] + return common.AuthInfoFromBytes(ret[:]) } func (dl *Publisher) calculateEncryptionKey() (*common.BlobKey, []byte) { diff --git a/pkg/internal/blobtypes/dynamiclink/publisher_test.go b/pkg/internal/blobtypes/dynamiclink/publisher_test.go index 2e84624..2e6b8f6 100644 --- a/pkg/internal/blobtypes/dynamiclink/publisher_test.go +++ b/pkg/internal/blobtypes/dynamiclink/publisher_test.go @@ -24,6 +24,7 @@ import ( "testing" "testing/iotest" + "github.com/cinode/go/pkg/common" "github.com/stretchr/testify/require" ) @@ -71,8 +72,10 @@ func TestFromAuthInfo(t *testing.T) { }) t.Run("Invalid auth info", func(t *testing.T) { - for i := 0; i < len(authInfo)-1; i++ { - dl2, err := FromAuthInfo(authInfo[:i]) + authInfoBytes := authInfo.Bytes() + for i := 0; i < len(authInfoBytes)-1; i++ { + brokenAuthInfo := common.AuthInfoFromBytes(authInfoBytes[:i]) + dl2, err := FromAuthInfo(brokenAuthInfo) require.ErrorIs(t, err, ErrInvalidDynamicLinkAuthInfo) require.Nil(t, dl2) } From 9ce175ceff49ac961ccb521e029c8107ed9592db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Sun, 29 Oct 2023 13:48:40 +0100 Subject: [PATCH 17/29] Shorten the cinodefs blackbox test --- pkg/cinodefs/cinodefs_blackbox_test.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/cinodefs/cinodefs_blackbox_test.go b/pkg/cinodefs/cinodefs_blackbox_test.go index 861a6ae..85225d7 100644 --- a/pkg/cinodefs/cinodefs_blackbox_test.go +++ b/pkg/cinodefs/cinodefs_blackbox_test.go @@ -100,11 +100,15 @@ func (c *CinodeFSMultiFileTestSuite) SetupTest() { require.NotNil(c.T(), fs) c.fs = fs - c.contentMap = make([]testFileEntry, 1000) - for i := 0; i < 1000; i++ { + const testFilesCount = 10 + const dirsCount = 7 + const subDirsCount = 19 + + c.contentMap = make([]testFileEntry, testFilesCount) + for i := 0; i < testFilesCount; i++ { c.contentMap[i].path = []string{ - fmt.Sprintf("dir%d", i%7), - fmt.Sprintf("subdir%d", i%19), + fmt.Sprintf("dir%d", i%dirsCount), + fmt.Sprintf("subdir%d", i%subDirsCount), fmt.Sprintf("file%d.txt", i), } c.contentMap[i].content = fmt.Sprintf("Hello world! from file %d!", i) From a341019612ea1d63eed8ace7eee1dfdd1bb88b8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Sun, 29 Oct 2023 13:50:24 +0100 Subject: [PATCH 18/29] Improve coverabe --- pkg/common/auth_info_test.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 pkg/common/auth_info_test.go diff --git a/pkg/common/auth_info_test.go b/pkg/common/auth_info_test.go new file mode 100644 index 0000000..a875052 --- /dev/null +++ b/pkg/common/auth_info_test.go @@ -0,0 +1,31 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAuthInfo(t *testing.T) { + authInfoBytes := []byte{1, 2, 3} + authInfo := AuthInfoFromBytes(authInfoBytes) + require.Equal(t, authInfoBytes, authInfo.Bytes()) + require.True(t, authInfo.Equal(AuthInfoFromBytes(authInfoBytes))) + require.Nil(t, new(BlobKey).Bytes()) +} From e06f5d309e64c2c35dae1be3fe16b0bcf2605a3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Sun, 29 Oct 2023 14:07:31 +0100 Subject: [PATCH 19/29] Move headwriter to separate internal util --- pkg/cinodefs/cinodefs.go | 7 +-- .../utilities/headwriter}/headwriter.go | 14 ++--- .../utilities/headwriter/headwriter_test.go | 52 +++++++++++++++++++ 3 files changed, 64 insertions(+), 9 deletions(-) rename pkg/{cinodefs => internal/utilities/headwriter}/headwriter.go (79%) create mode 100644 pkg/internal/utilities/headwriter/headwriter_test.go diff --git a/pkg/cinodefs/cinodefs.go b/pkg/cinodefs/cinodefs.go index 9501adb..c72d540 100644 --- a/pkg/cinodefs/cinodefs.go +++ b/pkg/cinodefs/cinodefs.go @@ -30,6 +30,7 @@ import ( "github.com/cinode/go/pkg/blobtypes" "github.com/cinode/go/pkg/common" "github.com/cinode/go/pkg/internal/blobtypes/dynamiclink" + "github.com/cinode/go/pkg/internal/utilities/headwriter" ) var ( @@ -199,11 +200,11 @@ func (fs *cinodeFS) createFileEntrypoint( data io.Reader, ep *Entrypoint, ) (*Entrypoint, error) { - var hw headWriter + var hw headwriter.Writer if ep.ep.MimeType == "" { // detect mimetype from the content - hw = newHeadWriter(512) + hw = headwriter.New(512) data = io.TeeReader(data, &hw) } @@ -213,7 +214,7 @@ func (fs *cinodeFS) createFileEntrypoint( } if ep.ep.MimeType == "" { - ep.ep.MimeType = http.DetectContentType(hw.data) + ep.ep.MimeType = http.DetectContentType(hw.Head()) } return setEntrypointBlobNameAndKey(bn, key, ep), nil diff --git a/pkg/cinodefs/headwriter.go b/pkg/internal/utilities/headwriter/headwriter.go similarity index 79% rename from pkg/cinodefs/headwriter.go rename to pkg/internal/utilities/headwriter/headwriter.go index 13f0973..4dfa6be 100644 --- a/pkg/cinodefs/headwriter.go +++ b/pkg/internal/utilities/headwriter/headwriter.go @@ -14,21 +14,21 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cinodefs +package headwriter -type headWriter struct { +type Writer struct { limit int data []byte } -func newHeadWriter(limit int) headWriter { - return headWriter{ +func New(limit int) Writer { + return Writer{ limit: limit, - data: make([]byte, limit), + data: make([]byte, 0, limit), } } -func (h *headWriter) Write(b []byte) (int, error) { +func (h *Writer) Write(b []byte) (int, error) { if len(h.data) >= h.limit { return len(b), nil } @@ -41,3 +41,5 @@ func (h *headWriter) Write(b []byte) (int, error) { h.data = append(h.data, b...) return len(b), nil } + +func (h *Writer) Head() []byte { return h.data } diff --git a/pkg/internal/utilities/headwriter/headwriter_test.go b/pkg/internal/utilities/headwriter/headwriter_test.go new file mode 100644 index 0000000..d524c68 --- /dev/null +++ b/pkg/internal/utilities/headwriter/headwriter_test.go @@ -0,0 +1,52 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package headwriter + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestHeadWriter(t *testing.T) { + lw := New(10) + + n, err := lw.Write([]byte{0, 1, 2, 3}) + require.NoError(t, err) + require.Equal(t, 4, n) + require.Equal(t, []byte{0, 1, 2, 3}, lw.Head()) + + n, err = lw.Write([]byte{}) + require.NoError(t, err) + require.Equal(t, 0, n) + require.Equal(t, []byte{0, 1, 2, 3}, lw.Head()) + + n, err = lw.Write([]byte{4, 5, 6, 7}) + require.NoError(t, err) + require.Equal(t, 4, n) + require.Equal(t, []byte{0, 1, 2, 3, 4, 5, 6, 7}, lw.Head()) + + n, err = lw.Write([]byte{8, 9, 10}) + require.NoError(t, err) + require.Equal(t, 3, n) + require.Equal(t, []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, lw.Head()) + + n, err = lw.Write([]byte{11, 12, 13, 14}) + require.NoError(t, err) + require.Equal(t, 4, n) + require.Equal(t, []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, lw.Head()) +} From 623d681952908fdac3b28c4d77d412d9e191738f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Sun, 29 Oct 2023 15:01:47 +0100 Subject: [PATCH 20/29] Add HTTP interface test --- pkg/cinodefs/httphandler/http.go | 58 +++--- pkg/cinodefs/httphandler/http_test.go | 245 ++++++++++++++++++++++++++ 2 files changed, 273 insertions(+), 30 deletions(-) create mode 100644 pkg/cinodefs/httphandler/http_test.go diff --git a/pkg/cinodefs/httphandler/http.go b/pkg/cinodefs/httphandler/http.go index c445d75..0a1d4c1 100644 --- a/pkg/cinodefs/httphandler/http.go +++ b/pkg/cinodefs/httphandler/http.go @@ -21,7 +21,6 @@ import ( "fmt" "io" "net/http" - "net/url" "strings" "github.com/cinode/go/pkg/cinodefs" @@ -41,34 +40,24 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { slog.String("Method", r.Method), ) - if r.Method != "GET" { + switch r.Method { + case "GET": + h.serveGet(w, r, log) + return + default: log.Error("Method not allowed") http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } +} +func (h *Handler) serveGet(w http.ResponseWriter, r *http.Request, log *slog.Logger) { path := r.URL.Path if strings.HasSuffix(path, "/") { path += h.IndexFile } pathList := strings.Split(strings.TrimPrefix(path, "/"), "/") - for i := range pathList { - p, err := url.PathUnescape(pathList[i]) - if err != nil { - log.WarnCtx(r.Context(), - "Incorrect request path", - "err", err, - ) - http.Error(w, - fmt.Sprintf("Could not unescape URL path segment: %s", err.Error()), - http.StatusBadRequest, - ) - return - } - pathList[i] = p - } - fileEP, err := h.FS.FindEntry(r.Context(), pathList) switch { case errors.Is(err, cinodefs.ErrEntryNotFound), @@ -76,31 +65,40 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { log.Warn("Not found") http.NotFound(w, r) return - case err != nil: - log.Error("Error serving request", "err", err) - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + case errors.Is(err, cinodefs.ErrModifiedDirectory): + // Can't get the entrypoint, but since it's a directory + // (only with unsaved changes), redirect to the directory itself + // that will in the end load the index file if present. + http.Redirect(w, r, r.URL.Path+"/", http.StatusTemporaryRedirect) + return + case h.handleHttpError(err, w, log, "Error finding entrypoint"): return } if fileEP.IsDir() { - http.Redirect(w, r, r.URL.Path+"/", http.StatusPermanentRedirect) + http.Redirect(w, r, r.URL.Path+"/", http.StatusTemporaryRedirect) return } - w.Header().Set("Content-Type", fileEP.MimeType()) rc, err := h.FS.OpenEntrypointData(r.Context(), fileEP) - if err != nil { - http.Error(w, - fmt.Sprintf("%s: %v", http.StatusText(http.StatusInternalServerError), err), - http.StatusInternalServerError, - ) - h.Log.Error("Error opening file", "err", err) + if h.handleHttpError(err, w, log, "Error opening file") { return } defer rc.Close() + w.Header().Set("Content-Type", fileEP.MimeType()) _, err = io.Copy(w, rc) + h.handleHttpError(err, w, log, "Error sending file") +} + +func (h *Handler) handleHttpError(err error, w http.ResponseWriter, log *slog.Logger, logMsg string) bool { if err != nil { - h.Log.Error("Error sending file", "err", err) + log.Error(logMsg, "err", err) + http.Error(w, + fmt.Sprintf("%s: %v", http.StatusText(http.StatusInternalServerError), err), + http.StatusInternalServerError, + ) + return true } + return false } diff --git a/pkg/cinodefs/httphandler/http_test.go b/pkg/cinodefs/httphandler/http_test.go new file mode 100644 index 0000000..25b5f9f --- /dev/null +++ b/pkg/cinodefs/httphandler/http_test.go @@ -0,0 +1,245 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package httphandler + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "testing/iotest" + + "github.com/cinode/go/pkg/blenc" + "github.com/cinode/go/pkg/cinodefs" + "github.com/cinode/go/pkg/common" + "github.com/cinode/go/pkg/datastore" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "golang.org/x/exp/slog" +) + +type mockDatastore struct { + datastore.DS + openFunc func(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) +} + +func (m *mockDatastore) Open(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { + if m.openFunc != nil { + return m.openFunc(ctx, name) + } + return m.DS.Open(ctx, name) +} + +type HandlerTestSuite struct { + suite.Suite + + ds mockDatastore + fs cinodefs.FS + handler *Handler + server *httptest.Server + logData *bytes.Buffer +} + +func TestHandlerTestSuite(t *testing.T) { + suite.Run(t, &HandlerTestSuite{}) +} + +func (s *HandlerTestSuite) SetupTest() { + s.ds = mockDatastore{DS: datastore.InMemory()} + fs, err := cinodefs.New( + context.Background(), + blenc.FromDatastore(&s.ds), + cinodefs.NewRootStaticDirectory(), + ) + require.NoError(s.T(), err) + s.fs = fs + + s.logData = bytes.NewBuffer(nil) + log := slog.New(slog.NewJSONHandler( + s.logData, + &slog.HandlerOptions{Level: slog.LevelDebug}, + )) + + s.handler = &Handler{ + FS: fs, + IndexFile: "index.html", + Log: log, + } + s.server = httptest.NewServer(s.handler) + s.T().Cleanup(s.server.Close) +} + +func (s *HandlerTestSuite) setEntry(t *testing.T, data string, path ...string) { + _, err := s.fs.SetEntryFile( + context.Background(), + path, + strings.NewReader(data), + ) + require.NoError(t, err) +} + +func (s *HandlerTestSuite) getEntry(t *testing.T, path string) (string, string, int) { + resp, err := http.Get(s.server.URL + path) + require.NoError(t, err) + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + return string(data), resp.Header.Get("content-type"), resp.StatusCode +} + +func (s *HandlerTestSuite) getData(t *testing.T, path string) string { + data, _, code := s.getEntry(t, path) + require.Equal(t, http.StatusOK, code) + return data +} + +func (s *HandlerTestSuite) TestSuccessfulFileDownload() { + s.setEntry(s.T(), "hello", "file.txt") + readBack := s.getData(s.T(), "/file.txt") + require.Equal(s.T(), "hello", readBack) +} + +func (s *HandlerTestSuite) TestNonGetRequest() { + t := s.T() + resp, err := http.Post(s.server.URL, "text/plain", strings.NewReader("Hello world!")) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode) +} + +func (s *HandlerTestSuite) TestNotFound() { + _, err := s.fs.SetEntryFile(context.Background(), []string{"hello.txt"}, strings.NewReader("hello")) + require.NoError(s.T(), err) + + _, _, code := s.getEntry(s.T(), "/no-hello.txt") + require.Equal(s.T(), http.StatusNotFound, code) + + _, _, code = s.getEntry(s.T(), "/hello.txt/world") + require.Equal(s.T(), http.StatusNotFound, code) +} + +func (s *HandlerTestSuite) TestReadIndexFile() { + s.setEntry(s.T(), "hello", "dir", "index.html") + + // Repeat twice, once before and once after flush + for i := 0; i < 2; i++ { + readBack := s.getData(s.T(), "/dir") + require.Equal(s.T(), "hello", readBack) + + err := s.fs.Flush(context.Background()) + require.NoError(s.T(), err) + } +} + +func (s *HandlerTestSuite) TestReadErrors() { + // Strictly controlled list of blob ids accessed, if at any time blob names + // would change, that would mean change in blob hashing algorithm + const bNameDir = "KAJgH9GYbmHxp4MUZvLswDh4t2TjTfVECAMmmv7MAzSZF" + const bNameFile = "pKFmwKyCeLeHjFRiwhGaajuhupPg5tS61tcL6F7sjBHRW" + + s.setEntry(s.T(), "hello", "file.txt") + + err := s.fs.Flush(context.Background()) + require.NoError(s.T(), err) + + s.T().Run("dir read error", func(t *testing.T) { + mockErr := errors.New("mock error dir") + s.ds.openFunc = func(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { + switch n := name.String(); n { + case bNameDir: + return nil, mockErr + case bNameFile: + return s.ds.DS.Open(ctx, name) + default: + panic("Unrecognized blob: " + n) + } + } + defer func() { s.ds.openFunc = nil }() + + _, _, code := s.getEntry(t, "/file.txt") + require.Equal(t, http.StatusInternalServerError, code) + require.Contains(t, s.logData.String(), mockErr.Error()) + }) + + s.T().Run("file open error", func(t *testing.T) { + mockErr := errors.New("mock error file open") + s.ds.openFunc = func(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { + switch n := name.String(); n { + case bNameDir: + return s.ds.DS.Open(ctx, name) + case bNameFile: + return nil, mockErr + default: + panic("Unrecognized blob: " + n) + } + } + defer func() { s.ds.openFunc = nil }() + + _, _, code := s.getEntry(t, "/file.txt") + require.Equal(t, http.StatusInternalServerError, code) + require.Contains(t, s.logData.String(), mockErr.Error()) + }) + + s.T().Run("file read error with error header", func(t *testing.T) { + mockErr := errors.New("mock error file read with headers") + s.ds.openFunc = func(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { + switch n := name.String(); n { + case bNameDir: + return s.ds.DS.Open(ctx, name) + case bNameFile: + return io.NopCloser(iotest.ErrReader(mockErr)), nil + default: + panic("Unrecognized blob: " + n) + } + } + defer func() { s.ds.openFunc = nil }() + + _, _, code := s.getEntry(t, "/file.txt") + require.Equal(t, http.StatusInternalServerError, code) + require.Contains(t, s.logData.String(), mockErr.Error()) + }) + + s.T().Run("file read error with partially sent data", func(t *testing.T) { + mockErr := errors.New("mock error file read without headers") + s.ds.openFunc = func(ctx context.Context, name *common.BlobName) (io.ReadCloser, error) { + switch n := name.String(); n { + case bNameDir: + return s.ds.DS.Open(ctx, name) + case bNameFile: + return io.NopCloser(io.MultiReader( + strings.NewReader("hello world!"), + iotest.ErrReader(mockErr), + )), nil + default: + panic("Unrecognized blob: " + n) + } + } + defer func() { s.ds.openFunc = nil }() + + content, _, _ := s.getEntry(t, "/file.txt") + // Since headers were already sent, there's no way to report back an error, + // we can only check if logs contain some error information + require.Contains(t, s.logData.String(), mockErr.Error()) + require.Contains(t, content, http.StatusText(http.StatusInternalServerError)) + }) +} From d71a8f51f771499628b80abc98fe2cd79e1f11b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Mon, 30 Oct 2023 10:46:34 +0100 Subject: [PATCH 21/29] Add directory uploader tests + small fixes --- pkg/cinodefs/cinodefs.go | 27 +++ pkg/cinodefs/uploader/directory.go | 39 ++-- pkg/cinodefs/uploader/directory_test.go | 286 +++++++++++++++++++++++ pkg/cinodefs/uploader/templates/dir.html | 2 +- pkg/cmd/cinode_web_proxy/root_test.go | 3 + pkg/cmd/static_datastore/compile.go | 5 + 6 files changed, 336 insertions(+), 26 deletions(-) create mode 100644 pkg/cinodefs/uploader/directory_test.go diff --git a/pkg/cinodefs/cinodefs.go b/pkg/cinodefs/cinodefs.go index c72d540..b7f3eae 100644 --- a/pkg/cinodefs/cinodefs.go +++ b/pkg/cinodefs/cinodefs.go @@ -31,6 +31,7 @@ import ( "github.com/cinode/go/pkg/common" "github.com/cinode/go/pkg/internal/blobtypes/dynamiclink" "github.com/cinode/go/pkg/internal/utilities/headwriter" + "github.com/cinode/go/pkg/utilities/golang" ) var ( @@ -49,6 +50,7 @@ var ( ErrCantReadDirectory = errors.New("can not read directory") ErrInvalidDirectoryData = errors.New("invalid directory data") ErrCantWriteDirectory = errors.New("can not write directory") + ErrMissingRootInfo = errors.New("root info not specified") ) const ( @@ -99,6 +101,11 @@ type FS interface { error, ) + OpenEntryData( + ctx context.Context, + path []string, + ) (io.ReadCloser, error) + OpenEntrypointData( ctx context.Context, ep *Entrypoint, @@ -151,6 +158,10 @@ func New( } } + if ret.rootEP == nil { + return nil, ErrMissingRootInfo + } + return &ret, nil } @@ -352,6 +363,22 @@ func (fs *cinodeFS) GenerateNewDynamicLinkEntrypoint() (*Entrypoint, error) { // } +func (fs *cinodeFS) OpenEntryData(ctx context.Context, path []string) (io.ReadCloser, error) { + ep, err := fs.FindEntry(ctx, path) + if err != nil { + return nil, err + } + if ep.IsDir() { + return nil, ErrCantReadDirectory + } + golang.Assert( + !ep.IsLink(), + "assumed that fs.FindEntry does not return a link", + ) + + return fs.OpenEntrypointData(ctx, ep) +} + func (fs *cinodeFS) OpenEntrypointData(ctx context.Context, ep *Entrypoint) (io.ReadCloser, error) { if ep == nil { return nil, ErrNilEntrypoint diff --git a/pkg/cinodefs/uploader/directory.go b/pkg/cinodefs/uploader/directory.go index 1b49dca..6500832 100644 --- a/pkg/cinodefs/uploader/directory.go +++ b/pkg/cinodefs/uploader/directory.go @@ -38,9 +38,10 @@ const ( ) var ( - ErrNotFound = blenc.ErrNotFound - ErrNotADirectory = errors.New("entry is not a directory") - ErrNotAFile = errors.New("entry is not a file") + ErrNotFound = blenc.ErrNotFound + ErrNotADirectory = errors.New("entry is not a directory") + ErrNotAFile = errors.New("entry is not a file") + ErrNotADirectoryOrAFile = errors.New("entry is neither a directory nor a regular file") ) func UploadStaticDirectory( @@ -56,9 +57,7 @@ func UploadStaticDirectory( log: slog.Default(), } for _, opt := range opts { - if err := opt(&c); err != nil { - return err - } + opt(&c) } _, err := c.compilePath(ctx, ".", c.basePath) @@ -66,28 +65,21 @@ func UploadStaticDirectory( return err } - err = cfs.Flush(ctx) - if err != nil { - return err - } - return nil } -type Option func(d *dirCompiler) error +type Option func(d *dirCompiler) -func BasePath(path []string) Option { - return Option(func(d *dirCompiler) error { +func BasePath(path ...string) Option { + return Option(func(d *dirCompiler) { d.basePath = path - return nil }) } func CreateIndexFile(indexFile string) Option { - return Option(func(d *dirCompiler) error { + return Option(func(d *dirCompiler) { d.createIndexFile = true d.indexFileName = indexFile - return nil }) } @@ -151,7 +143,7 @@ func (d *dirCompiler) compilePath( } d.log.ErrorContext(ctx, "path is neither dir nor a regular file", "path", srcPath) - return nil, fmt.Errorf("neither dir nor a regular file: %v", srcPath) + return nil, fmt.Errorf("%w: %v", ErrNotADirectoryOrAFile, srcPath) } func (d *dirCompiler) compileFile(ctx context.Context, srcPath string, dstPath []string) (string, error) { @@ -178,9 +170,6 @@ func (d *dirCompiler) compileDir(ctx context.Context, srcPath string, dstPath [] return 0, fmt.Errorf("couldn't read contents of dir %v: %w", srcPath, err) } - // TODO: Reset directory content - // TODO: Build index file - entries := make([]*dirEntry, 0, len(fileList)) hasIndex := false @@ -207,9 +196,7 @@ func (d *dirCompiler) compileDir(ctx context.Context, srcPath string, dstPath [] "entries": entries, "indexName": d.indexFileName, }) - if err != nil { - return 0, err - } + golang.Assert(err == nil, "template execution must not fail") _, err = d.cfs.SetEntryFile(ctx, append(dstPath, d.indexFileName), @@ -226,5 +213,7 @@ func (d *dirCompiler) compileDir(ctx context.Context, srcPath string, dstPath [] //go:embed templates/dir.html var _dirIndexTemplateStr string var dirIndexTemplate = golang.Must( - template.New("dir").Parse(_dirIndexTemplateStr), + template. + New("dir"). + Parse(_dirIndexTemplateStr), ) diff --git a/pkg/cinodefs/uploader/directory_test.go b/pkg/cinodefs/uploader/directory_test.go new file mode 100644 index 0000000..d6162f3 --- /dev/null +++ b/pkg/cinodefs/uploader/directory_test.go @@ -0,0 +1,286 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package uploader_test + +import ( + "context" + "errors" + "io" + "io/fs" + "strings" + "testing" + "testing/fstest" + + "github.com/cinode/go/pkg/blenc" + "github.com/cinode/go/pkg/cinodefs" + "github.com/cinode/go/pkg/cinodefs/uploader" + "github.com/cinode/go/pkg/datastore" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type DirectoryTestSuite struct { + suite.Suite + + cfs cinodefs.FS +} + +func TestDirectoryTestSuite(t *testing.T) { + suite.Run(t, &DirectoryTestSuite{}) +} + +func (s *DirectoryTestSuite) SetupTest() { + cfs, err := cinodefs.New( + context.Background(), + blenc.FromDatastore(datastore.InMemory()), + cinodefs.NewRootStaticDirectory(), + ) + require.NoError(s.T(), err) + s.cfs = cfs +} + +func (s *DirectoryTestSuite) singleFileFs() fstest.MapFS { + return fstest.MapFS{ + "file.txt": &fstest.MapFile{Data: []byte("hello")}, + } +} + +type wrapFS struct { + fs.FS + + openFunc func(path string) (fs.File, error) + statFunc func(name string) (fs.FileInfo, error) + readDirFunc func(name string) ([]fs.DirEntry, error) +} + +func (w *wrapFS) Open(path string) (fs.File, error) { + if w.openFunc != nil { + return w.openFunc(path) + } + return w.FS.Open(path) +} + +func (w *wrapFS) Stat(name string) (fs.FileInfo, error) { + if w.statFunc != nil { + return w.statFunc(name) + } + return fs.Stat(w.FS, name) +} + +func (w *wrapFS) ReadDir(name string) ([]fs.DirEntry, error) { + if w.readDirFunc != nil { + return w.readDirFunc(name) + } + return fs.ReadDir(w.FS, name) +} + +func (s *DirectoryTestSuite) uploadFS(t *testing.T, fs fs.FS, opts ...uploader.Option) { + err := uploader.UploadStaticDirectory( + context.Background(), + fs, + s.cfs, + opts..., + ) + require.NoError(t, err) +} + +func (s *DirectoryTestSuite) readContent(t *testing.T, path ...string) (string, error) { + rc, err := s.cfs.OpenEntryData(context.Background(), path) + if err != nil { + return "", err + } + defer rc.Close() + data, err := io.ReadAll(rc) + return string(data), err +} + +func (s *DirectoryTestSuite) TestSingleFileUploadDefaultOptions() { + s.uploadFS(s.T(), s.singleFileFs()) + + readBack, err := s.readContent(s.T(), "file.txt") + require.NoError(s.T(), err) + require.Equal(s.T(), "hello", readBack) +} + +func (s *DirectoryTestSuite) TestSingleFileUploadBasePath() { + s.uploadFS(s.T(), s.singleFileFs(), uploader.BasePath("sub", "dir")) + + readBack, err := s.readContent(s.T(), "sub", "dir", "file.txt") + require.NoError(s.T(), err) + require.Equal(s.T(), "hello", readBack) + + _, err = s.readContent(s.T(), "file.txt") + require.ErrorIs(s.T(), err, cinodefs.ErrEntryNotFound) +} + +func (s *DirectoryTestSuite) TestSingleFileUploadWithIndexFile() { + s.uploadFS(s.T(), s.singleFileFs(), uploader.CreateIndexFile("index.html")) + + readBack, err := s.readContent(s.T(), "index.html") + require.NoError(s.T(), err) + require.True(s.T(), strings.HasPrefix(readBack, " diff --git a/pkg/cmd/cinode_web_proxy/root_test.go b/pkg/cmd/cinode_web_proxy/root_test.go index 908affa..afb1930 100644 --- a/pkg/cmd/cinode_web_proxy/root_test.go +++ b/pkg/cmd/cinode_web_proxy/root_test.go @@ -171,6 +171,9 @@ func TestWebProxyHandlerSimplePage(t *testing.T) { ) require.NoError(t, err) + err = fs.Flush(context.Background()) + require.NoError(t, err) + ep, err := fs.RootEntrypoint() require.NoError(t, err) return ep diff --git a/pkg/cmd/static_datastore/compile.go b/pkg/cmd/static_datastore/compile.go index 2de246b..363b231 100644 --- a/pkg/cmd/static_datastore/compile.go +++ b/pkg/cmd/static_datastore/compile.go @@ -219,6 +219,11 @@ func compileFS( return nil, nil, fmt.Errorf("couldn't upload directory content: %w", err) } + err = fs.Flush(ctx) + if err != nil { + return nil, nil, fmt.Errorf("couldn't flush after directory upload: %w", err) + } + ep, err := fs.RootEntrypoint() if err != nil { return nil, nil, fmt.Errorf("couldn't get root entrypoint from cinodefs instance: %w", err) From 1287a9a63de9c7e3ac068c0168867374c1b1da08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Mon, 30 Oct 2023 20:31:20 +0100 Subject: [PATCH 22/29] Remove file no longer needed --- pkg/cinodefs/internal/protobuf/protobuf.go | 94 ---------------------- 1 file changed, 94 deletions(-) delete mode 100644 pkg/cinodefs/internal/protobuf/protobuf.go diff --git a/pkg/cinodefs/internal/protobuf/protobuf.go b/pkg/cinodefs/internal/protobuf/protobuf.go deleted file mode 100644 index 79593d3..0000000 --- a/pkg/cinodefs/internal/protobuf/protobuf.go +++ /dev/null @@ -1,94 +0,0 @@ -/* -Copyright © 2023 Bartłomiej Święcki (byo) - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package protobuf - -//go:generate protoc --go_out=. protobuf.proto - -import ( - "errors" - "time" - - "github.com/cinode/go/pkg/common" - "google.golang.org/protobuf/proto" -) - -var ( - ErrInvalidEntrypoint = errors.New("invalid entrypoint") - ErrInvalidEntrypointTime = errors.New("%w: time validation failed") -) - -func (ep *Entrypoint) ToBytes() ([]byte, error) { - return proto.Marshal(ep) -} - -func EntryPointFromBytes(b []byte) (*Entrypoint, error) { - ret := &Entrypoint{} - err := proto.Unmarshal(b, ret) - if err != nil { - return nil, err - } - return ret, nil -} - -func (wi *WriterInfo) ToBytes() ([]byte, error) { - return proto.Marshal(wi) -} - -func WriterInfoFromBytes(b []byte) (*WriterInfo, error) { - ret := &WriterInfo{} - err := proto.Unmarshal(b, ret) - if err != nil { - return nil, err - } - return ret, nil -} - -func (ep *Entrypoint) Validate(currentTime time.Time) error { - currentTimeMicro := currentTime.UnixMicro() - - if ep.GetNotValidAfterUnixMicro() != 0 && - currentTimeMicro > ep.GetNotValidAfterUnixMicro() { - return ErrInvalidEntrypointTime - } - - if ep.GetNotValidBeforeUnixMicro() != 0 && - currentTimeMicro < ep.GetNotValidBeforeUnixMicro() { - return ErrInvalidEntrypointTime - } - - return nil -} - -func (ep *Entrypoint) ValidateAndParse(currentTime time.Time) ( - *common.BlobName, - *common.BlobKey, - error, -) { - err := ep.Validate(currentTime) - if err != nil { - return nil, nil, err - } - - bn, err := common.BlobNameFromBytes(ep.BlobName) - if err != nil { - return nil, nil, err - } - - key := common.BlobKeyFromBytes(ep.GetKeyInfo().GetKey()) - - return bn, key, nil -} From afa4ac359064d87e8f240c7a8badb72e2bda61c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Mon, 30 Oct 2023 23:25:26 +0100 Subject: [PATCH 23/29] Improve test coverage and fix discovered issues --- pkg/cinodefs/cinodefs_blackbox_test.go | 358 ----------- .../{cinodefs.go => cinodefs_interface.go} | 102 ++- pkg/cinodefs/cinodefs_interface_bb_test.go | 603 ++++++++++++++++++ pkg/cinodefs/cinodefs_options.go | 43 +- pkg/cinodefs/cinodefs_options_bb_test.go | 115 ++++ pkg/cinodefs/cinodefs_traverse.go | 8 +- pkg/cinodefs/context.go | 6 +- pkg/cinodefs/entrypoint_bb_test.go | 77 +++ pkg/cinodefs/entrypoint_options.go | 15 +- pkg/cinodefs/node_link.go | 2 +- pkg/cinodefs/writerinfo_bb_test.go | 42 ++ 11 files changed, 962 insertions(+), 409 deletions(-) delete mode 100644 pkg/cinodefs/cinodefs_blackbox_test.go rename pkg/cinodefs/{cinodefs.go => cinodefs_interface.go} (83%) create mode 100644 pkg/cinodefs/cinodefs_interface_bb_test.go create mode 100644 pkg/cinodefs/cinodefs_options_bb_test.go create mode 100644 pkg/cinodefs/entrypoint_bb_test.go create mode 100644 pkg/cinodefs/writerinfo_bb_test.go diff --git a/pkg/cinodefs/cinodefs_blackbox_test.go b/pkg/cinodefs/cinodefs_blackbox_test.go deleted file mode 100644 index 85225d7..0000000 --- a/pkg/cinodefs/cinodefs_blackbox_test.go +++ /dev/null @@ -1,358 +0,0 @@ -/* -Copyright © 2023 Bartłomiej Święcki (byo) - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cinodefs_test - -import ( - "context" - "fmt" - "io" - "strings" - "testing" - - "github.com/cinode/go/pkg/blenc" - "github.com/cinode/go/pkg/cinodefs" - "github.com/cinode/go/pkg/datastore" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" -) - -func TestCinodeFSSingleFileScenario(t *testing.T) { - ctx := context.Background() - fs, err := cinodefs.New(ctx, - blenc.FromDatastore(datastore.InMemory()), - cinodefs.NewRootDynamicLink(), - ) - require.NoError(t, err) - require.NotNil(t, fs) - - { // Check single file write operation - path1 := []string{"dir", "subdir", "file.txt"} - - ep1, err := fs.SetEntryFile(ctx, - path1, - strings.NewReader("Hello world!"), - ) - require.NoError(t, err) - require.NotNil(t, ep1) - - ep2, err := fs.FindEntry( - ctx, - path1, - ) - require.NoError(t, err) - require.NotNil(t, ep2) - - require.Equal(t, ep1.String(), ep2.String()) - - // Directories are modified, not yet flushed - for i := range path1 { - ep3, err := fs.FindEntry(ctx, path1[:i]) - require.ErrorIs(t, err, cinodefs.ErrModifiedDirectory) - require.Nil(t, ep3) - } - - err = fs.Flush(ctx) - require.NoError(t, err) - } -} - -type testFileEntry struct { - path []string - content string - mimeType string -} - -type CinodeFSMultiFileTestSuite struct { - suite.Suite - - ds datastore.DS - fs cinodefs.FS - contentMap []testFileEntry -} - -func TestCinodeFSMultiFileTestSuite(t *testing.T) { - suite.Run(t, &CinodeFSMultiFileTestSuite{}) -} - -func (c *CinodeFSMultiFileTestSuite) SetupTest() { - ctx := context.Background() - - c.ds = datastore.InMemory() - fs, err := cinodefs.New(ctx, - blenc.FromDatastore(c.ds), - cinodefs.NewRootDynamicLink(), - ) - require.NoError(c.T(), err) - require.NotNil(c.T(), fs) - c.fs = fs - - const testFilesCount = 10 - const dirsCount = 7 - const subDirsCount = 19 - - c.contentMap = make([]testFileEntry, testFilesCount) - for i := 0; i < testFilesCount; i++ { - c.contentMap[i].path = []string{ - fmt.Sprintf("dir%d", i%dirsCount), - fmt.Sprintf("subdir%d", i%subDirsCount), - fmt.Sprintf("file%d.txt", i), - } - c.contentMap[i].content = fmt.Sprintf("Hello world! from file %d!", i) - c.contentMap[i].mimeType = "text/plain" - } - - for _, file := range c.contentMap { - _, err := c.fs.SetEntryFile(ctx, - file.path, - strings.NewReader(file.content), - ) - require.NoError(c.T(), err) - } - - c.checkContentMap(fs) - - err = c.fs.Flush(context.Background()) - require.NoError(c.T(), err) - - c.checkContentMap(c.fs) -} - -func (c *CinodeFSMultiFileTestSuite) checkContentMap(fs cinodefs.FS) { - ctx := context.Background() - for _, file := range c.contentMap { - ep, err := fs.FindEntry(ctx, file.path) - require.NoError(c.T(), err) - require.Contains(c.T(), ep.MimeType(), file.mimeType) - - rc, err := fs.OpenEntrypointData(ctx, ep) - require.NoError(c.T(), err) - defer rc.Close() - - data, err := io.ReadAll(rc) - require.NoError(c.T(), err) - - require.Equal(c.T(), file.content, string(data)) - } -} - -func (c *CinodeFSMultiFileTestSuite) TestReopeningInReadOnlyMode() { - ctx := context.Background() - rootEP, err := c.fs.RootEntrypoint() - require.NoError(c.T(), err) - - fs2, err := cinodefs.New( - ctx, - blenc.FromDatastore(c.ds), - cinodefs.RootEntrypoint(rootEP), - ) - require.NoError(c.T(), err) - require.NotNil(c.T(), fs2) - - c.checkContentMap(fs2) - - _, err = c.fs.SetEntryFile(ctx, - c.contentMap[0].path, - strings.NewReader("modified content"), - ) - require.NoError(c.T(), err) - - // Data in fs was not yet flushed to the datastore, fs2 should still refer to the old content - c.checkContentMap(fs2) - - err = c.fs.Flush(ctx) - require.NoError(c.T(), err) - - // reopen fs2 to avoid any caching issues - fs2, err = cinodefs.New( - ctx, - blenc.FromDatastore(c.ds), - cinodefs.RootEntrypoint(rootEP), - ) - require.NoError(c.T(), err) - - // Check with modified content map - c.contentMap[0].content = "modified content" - c.checkContentMap(fs2) - - // We should not be allowed to modify fs2 without writer info - ep, err := fs2.SetEntryFile(ctx, c.contentMap[0].path, strings.NewReader("should fail")) - require.ErrorIs(c.T(), err, cinodefs.ErrMissingWriterInfo) - require.Nil(c.T(), ep) - c.checkContentMap(c.fs) - c.checkContentMap(fs2) -} - -func (c *CinodeFSMultiFileTestSuite) TestReopeningInReadWriteMode() { - ctx := context.Background() - - rootWriterInfo, err := c.fs.RootWriterInfo(ctx) - require.NoError(c.T(), err) - require.NotNil(c.T(), rootWriterInfo) - - fs3, err := cinodefs.New( - ctx, - blenc.FromDatastore(c.ds), - cinodefs.RootWriterInfo(rootWriterInfo), - ) - require.NoError(c.T(), err) - require.NotNil(c.T(), fs3) - - c.checkContentMap(fs3) - - // With a proper auth info we can modify files in the root path - ep, err := fs3.SetEntryFile(ctx, c.contentMap[0].path, strings.NewReader("modified through fs3")) - require.NoError(c.T(), err) - require.NotNil(c.T(), ep) - - c.contentMap[0].content = "modified through fs3" - c.checkContentMap(fs3) -} - -func (c *CinodeFSMultiFileTestSuite) TestRemovalOfAFile() { - ctx := context.Background() - - err := c.fs.DeleteEntry(ctx, c.contentMap[0].path) - require.NoError(c.T(), err) - - c.contentMap = c.contentMap[1:] - c.checkContentMap(c.fs) -} - -func (c *CinodeFSMultiFileTestSuite) TestRemovalOfADirectory() { - ctx := context.Background() - - removedPath := c.contentMap[0].path[:2] - - err := c.fs.DeleteEntry(ctx, removedPath) - require.NoError(c.T(), err) - - filteredEntries := []testFileEntry{} - removed := 0 - for _, e := range c.contentMap { - if e.path[0] == removedPath[0] && e.path[1] == removedPath[1] { - continue - } - - filteredEntries = append(filteredEntries, e) - removed++ - } - c.contentMap = filteredEntries - require.NotZero(c.T(), removed) - - c.checkContentMap(c.fs) - - err = c.fs.DeleteEntry(ctx, removedPath) - require.ErrorIs(c.T(), err, cinodefs.ErrEntryNotFound) - - c.checkContentMap(c.fs) -} - -func (c *CinodeFSMultiFileTestSuite) TestDeleteTreatFileAsDirectory() { - ctx := context.Background() - - path := append(c.contentMap[0].path, "sub-file") - err := c.fs.DeleteEntry(ctx, path) - require.ErrorIs(c.T(), err, cinodefs.ErrNotADirectory) -} - -func (c *CinodeFSMultiFileTestSuite) TestPreventSettingFileAsDirectory() { - ctx := context.Background() - - path := append(c.contentMap[0].path, "sub-file") - _, err := c.fs.SetEntryFile(ctx, path, strings.NewReader("should not happen")) - require.ErrorIs(c.T(), err, cinodefs.ErrNotADirectory) -} - -func (c *CinodeFSMultiFileTestSuite) TestPreventSettingEmptyEntryName() { - ctx := context.Background() - - for _, path := range [][]string{ - {"", "subdir", "file.txt"}, - {"dir", "", "file.txt"}, - {"dir", "subdir", ""}, - } { - c.T().Run(strings.Join(path, "::"), func(t *testing.T) { - _, err := c.fs.SetEntryFile(ctx, path, strings.NewReader("should not succeed")) - require.ErrorIs(t, err, cinodefs.ErrEmptyName) - - }) - } - -} - -func (c *CinodeFSMultiFileTestSuite) TestRootEPLinkOnDirtyFS() { - ctx := context.Background() - - ep1, err := c.fs.RootEntrypoint() - require.NoError(c.T(), err) - - _, err = c.fs.SetEntryFile(ctx, c.contentMap[0].path, strings.NewReader("hello")) - require.NoError(c.T(), err) - - ep2, err := c.fs.RootEntrypoint() - require.NoError(c.T(), err) - - // Even though dirty, entrypoint won't change it's content - require.Equal(c.T(), ep1.String(), ep2.String()) - - err = c.fs.Flush(ctx) - require.NoError(c.T(), err) - - ep3, err := c.fs.RootEntrypoint() - require.NoError(c.T(), err) - - require.Equal(c.T(), ep1.String(), ep3.String()) -} - -func (c *CinodeFSMultiFileTestSuite) TestRootEPDirectoryOnDirtyFS() { - ctx := context.Background() - - rootDir, err := c.fs.FindEntry(ctx, []string{}) - require.NoError(c.T(), err) - - fs2, err := cinodefs.New(ctx, - blenc.FromDatastore(c.ds), - cinodefs.RootEntrypoint(rootDir), - ) - require.NoError(c.T(), err) - - ep1, err := fs2.RootEntrypoint() - require.NoError(c.T(), err) - require.Equal(c.T(), rootDir.String(), ep1.String()) - - _, err = fs2.SetEntryFile(ctx, c.contentMap[0].path, strings.NewReader("hello")) - require.NoError(c.T(), err) - - ep2, err := fs2.RootEntrypoint() - require.ErrorIs(c.T(), err, cinodefs.ErrModifiedDirectory) - require.Nil(c.T(), ep2) - - err = fs2.Flush(ctx) - require.NoError(c.T(), err) - - ep3, err := c.fs.RootEntrypoint() - require.NoError(c.T(), err) - - require.NotEqual(c.T(), ep1.String(), ep3.String()) -} - -func (c *CinodeFSMultiFileTestSuite) TestWriteOnlyLink() { - // ctx := context.Background() - // fs, err := graph.NewCinodeFS(ctx, blenc.FromDatastore(datastore.InMemory()), graph.NewRootDynamicLink()) - // require.NoError(c.T(), err) - -} diff --git a/pkg/cinodefs/cinodefs.go b/pkg/cinodefs/cinodefs_interface.go similarity index 83% rename from pkg/cinodefs/cinodefs.go rename to pkg/cinodefs/cinodefs_interface.go index b7f3eae..bb4f504 100644 --- a/pkg/cinodefs/cinodefs.go +++ b/pkg/cinodefs/cinodefs_interface.go @@ -47,7 +47,7 @@ var ( ErrEmptyName = errors.New("entry name can not be empty") ErrDuplicateEntry = errors.New("duplicate entry") ErrEntryNotFound = errors.New("entry not found") - ErrCantReadDirectory = errors.New("can not read directory") + ErrIsADirectory = errors.New("entry is a directory") ErrInvalidDirectoryData = errors.New("invalid directory data") ErrCantWriteDirectory = errors.New("can not write directory") ErrMissingRootInfo = errors.New("root info not specified") @@ -96,8 +96,11 @@ type FS interface { path []string, ) error - GenerateNewDynamicLinkEntrypoint() ( - *Entrypoint, + InjectDynamicLink( + ctx context.Context, + path []string, + ) ( + *WriterInfo, error, ) @@ -171,16 +174,13 @@ func (fs *cinodeFS) SetEntryFile( data io.Reader, opts ...EntrypointOption, ) (*Entrypoint, error) { - ep, err := entrypointFromOptions(ctx, opts...) - if err != nil { - return nil, err - } + ep := entrypointFromOptions(ctx, opts...) if ep.ep.MimeType == "" && len(path) > 0 { // Try detecting mime type from filename extension ep.ep.MimeType = mime.TypeByExtension(filepath.Ext(path[len(path)-1])) } - ep, err = fs.createFileEntrypoint(ctx, data, ep) + ep, err := fs.createFileEntrypoint(ctx, data, ep) if err != nil { return nil, err } @@ -198,11 +198,7 @@ func (fs *cinodeFS) CreateFileEntrypoint( data io.Reader, opts ...EntrypointOption, ) (*Entrypoint, error) { - ep, err := entrypointFromOptions(ctx, opts...) - if err != nil { - return nil, err - } - + ep := entrypointFromOptions(ctx, opts...) return fs.createFileEntrypoint(ctx, data, ep) } @@ -251,8 +247,8 @@ func (fs *cinodeFS) SetEntry( ctx, path, traverseOptions{ - createNodes: true, - maxLinkDepth: fs.maxLinkRedirects, + createNodes: true, + maxLinkRedirects: fs.maxLinkRedirects, }, whenReached, ) @@ -277,8 +273,8 @@ func (fs *cinodeFS) ResetDir(ctx context.Context, path []string) error { ctx, path, traverseOptions{ - createNodes: true, - maxLinkDepth: fs.maxLinkRedirects, + createNodes: true, + maxLinkRedirects: fs.maxLinkRedirects, }, whenReached, ) @@ -344,32 +340,86 @@ func (fs *cinodeFS) DeleteEntry(ctx context.Context, path []string) error { ) } -func (fs *cinodeFS) GenerateNewDynamicLinkEntrypoint() (*Entrypoint, error) { +func (fs *cinodeFS) InjectDynamicLink( + ctx context.Context, + path []string, +) ( + *WriterInfo, + error, +) { + var retWi *WriterInfo + + whenReached := func( + ctx context.Context, + current node, + isWriteable bool, + ) (node, dirtyState, error) { + if !isWriteable { + return nil, 0, ErrMissingWriterInfo + } + + ep, ai, err := fs.generateNewDynamicLinkEntrypoint() + if err != nil { + return nil, 0, err + } + + key, err := fs.c.keyFromEntrypoint(ctx, ep) + if err != nil { + return nil, 0, err + } + + retWi = writerInfoFromBlobNameKeyAndAuthInfo(ep.BlobName(), key, ai) + return &nodeLink{ + ep: ep, + target: current, + // Link itself must be marked as dirty - even if the content is clean, + // the link itself must be persisted + dState: dsSubDirty, + }, + // Parent node becomes dirty - new link is a new blob + dsDirty, + nil + } + + err := fs.traverseGraph( + ctx, + path, + traverseOptions{ + createNodes: true, + maxLinkRedirects: fs.maxLinkRedirects, + }, + whenReached, + ) + if err != nil { + return nil, err + } + + return retWi, nil +} + +func (fs *cinodeFS) generateNewDynamicLinkEntrypoint() (*Entrypoint, *common.AuthInfo, error) { // Generate new entrypoint link data but do not yet store it in datastore link, err := dynamiclink.Create(fs.randSource) if err != nil { - return nil, err + return nil, nil, err } bn := link.BlobName() key := link.EncryptionKey() + ai := link.AuthInfo() - fs.c.authInfos[bn.String()] = link.AuthInfo() + fs.c.authInfos[bn.String()] = ai - return EntrypointFromBlobNameAndKey(bn, key), nil + return EntrypointFromBlobNameAndKey(bn, key), ai, nil } -// func (fs *cinodeFS) ReplacePathWithLink(ctx context.Context, path []string) (WriterInfo, error) { - -// } - func (fs *cinodeFS) OpenEntryData(ctx context.Context, path []string) (io.ReadCloser, error) { ep, err := fs.FindEntry(ctx, path) if err != nil { return nil, err } if ep.IsDir() { - return nil, ErrCantReadDirectory + return nil, ErrIsADirectory } golang.Assert( !ep.IsLink(), diff --git a/pkg/cinodefs/cinodefs_interface_bb_test.go b/pkg/cinodefs/cinodefs_interface_bb_test.go new file mode 100644 index 0000000..1f56727 --- /dev/null +++ b/pkg/cinodefs/cinodefs_interface_bb_test.go @@ -0,0 +1,603 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cinodefs_test + +import ( + "context" + "crypto/rand" + "fmt" + "io" + "strings" + "testing" + "time" + + "github.com/cinode/go/pkg/blenc" + "github.com/cinode/go/pkg/cinodefs" + "github.com/cinode/go/pkg/datastore" + "github.com/cinode/go/pkg/internal/blobtypes/dynamiclink" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +func TestCinodeFSSingleFileScenario(t *testing.T) { + ctx := context.Background() + fs, err := cinodefs.New(ctx, + blenc.FromDatastore(datastore.InMemory()), + cinodefs.NewRootDynamicLink(), + ) + require.NoError(t, err) + require.NotNil(t, fs) + + { // Check single file write operation + path1 := []string{"dir", "subdir", "file.txt"} + + ep1, err := fs.SetEntryFile(ctx, + path1, + strings.NewReader("Hello world!"), + ) + require.NoError(t, err) + require.NotNil(t, ep1) + + ep2, err := fs.FindEntry( + ctx, + path1, + ) + require.NoError(t, err) + require.NotNil(t, ep2) + + require.Equal(t, ep1.String(), ep2.String()) + + // Directories are modified, not yet flushed + for i := range path1 { + ep3, err := fs.FindEntry(ctx, path1[:i]) + require.ErrorIs(t, err, cinodefs.ErrModifiedDirectory) + require.Nil(t, ep3) + } + + err = fs.Flush(ctx) + require.NoError(t, err) + } +} + +type testFileEntry struct { + path []string + content string + mimeType string +} + +type CinodeFSMultiFileTestSuite struct { + suite.Suite + + ds datastore.DS + fs cinodefs.FS + contentMap []testFileEntry + maxLinkRedirects int + timeFunc func() time.Time +} + +func TestCinodeFSMultiFileTestSuite(t *testing.T) { + suite.Run(t, &CinodeFSMultiFileTestSuite{ + maxLinkRedirects: 5, + }) +} + +func (c *CinodeFSMultiFileTestSuite) SetupTest() { + ctx := context.Background() + + c.timeFunc = time.Now + c.ds = datastore.InMemory() + fs, err := cinodefs.New(ctx, + blenc.FromDatastore(c.ds), + cinodefs.NewRootDynamicLink(), + cinodefs.MaxLinkRedirects(c.maxLinkRedirects), + cinodefs.TimeFunc(c.timeFunc), + ) + require.NoError(c.T(), err) + require.NotNil(c.T(), fs) + c.fs = fs + + const testFilesCount = 10 + const dirsCount = 3 + const subDirsCount = 2 + + c.contentMap = make([]testFileEntry, testFilesCount) + for i := 0; i < testFilesCount; i++ { + c.contentMap[i].path = []string{ + fmt.Sprintf("dir%d", i%dirsCount), + fmt.Sprintf("subdir%d", i%subDirsCount), + fmt.Sprintf("file%d.txt", i), + } + c.contentMap[i].content = fmt.Sprintf("Hello world! from file %d!", i) + c.contentMap[i].mimeType = "text/plain" + } + + for _, file := range c.contentMap { + _, err := c.fs.SetEntryFile(ctx, + file.path, + strings.NewReader(file.content), + ) + require.NoError(c.T(), err) + } + + err = c.fs.Flush(context.Background()) + require.NoError(c.T(), err) +} + +func (c *CinodeFSMultiFileTestSuite) checkContentMap(t *testing.T, fs cinodefs.FS) { + ctx := context.Background() + for _, file := range c.contentMap { + ep, err := fs.FindEntry(ctx, file.path) + require.NoError(t, err) + require.Contains(t, ep.MimeType(), file.mimeType) + + rc, err := fs.OpenEntrypointData(ctx, ep) + require.NoError(t, err) + defer rc.Close() + + data, err := io.ReadAll(rc) + require.NoError(t, err) + + require.Equal(t, file.content, string(data)) + } +} + +func (c *CinodeFSMultiFileTestSuite) TestReopeningInReadOnlyMode() { + ctx := context.Background() + rootEP, err := c.fs.RootEntrypoint() + require.NoError(c.T(), err) + + fs2, err := cinodefs.New( + ctx, + blenc.FromDatastore(c.ds), + cinodefs.RootEntrypointString(rootEP.String()), + ) + require.NoError(c.T(), err) + require.NotNil(c.T(), fs2) + + c.checkContentMap(c.T(), fs2) + + _, err = c.fs.SetEntryFile(ctx, + c.contentMap[0].path, + strings.NewReader("modified content"), + ) + require.NoError(c.T(), err) + + // Data in fs was not yet flushed to the datastore, fs2 should still refer to the old content + c.checkContentMap(c.T(), fs2) + + err = c.fs.Flush(ctx) + require.NoError(c.T(), err) + + // reopen fs2 to avoid any caching issues + fs2, err = cinodefs.New( + ctx, + blenc.FromDatastore(c.ds), + cinodefs.RootEntrypoint(rootEP), + ) + require.NoError(c.T(), err) + + // Check with modified content map + c.contentMap[0].content = "modified content" + c.checkContentMap(c.T(), fs2) + + // We should not be allowed to modify fs2 without writer info + ep, err := fs2.SetEntryFile(ctx, c.contentMap[0].path, strings.NewReader("should fail")) + require.ErrorIs(c.T(), err, cinodefs.ErrMissingWriterInfo) + require.Nil(c.T(), ep) + c.checkContentMap(c.T(), c.fs) + c.checkContentMap(c.T(), fs2) +} + +func (c *CinodeFSMultiFileTestSuite) TestReopeningInReadWriteMode() { + ctx := context.Background() + + rootWriterInfo, err := c.fs.RootWriterInfo(ctx) + require.NoError(c.T(), err) + require.NotNil(c.T(), rootWriterInfo) + + fs3, err := cinodefs.New( + ctx, + blenc.FromDatastore(c.ds), + cinodefs.RootWriterInfoString(rootWriterInfo.String()), + ) + require.NoError(c.T(), err) + require.NotNil(c.T(), fs3) + + c.checkContentMap(c.T(), fs3) + + // With a proper auth info we can modify files in the root path + ep, err := fs3.SetEntryFile(ctx, c.contentMap[0].path, strings.NewReader("modified through fs3")) + require.NoError(c.T(), err) + require.NotNil(c.T(), ep) + + c.contentMap[0].content = "modified through fs3" + c.checkContentMap(c.T(), fs3) +} + +func (c *CinodeFSMultiFileTestSuite) TestRemovalOfAFile() { + ctx := context.Background() + + err := c.fs.DeleteEntry(ctx, c.contentMap[0].path) + require.NoError(c.T(), err) + + c.contentMap = c.contentMap[1:] + c.checkContentMap(c.T(), c.fs) +} + +func (c *CinodeFSMultiFileTestSuite) TestRemovalOfADirectory() { + ctx := context.Background() + + removedPath := c.contentMap[0].path[:2] + + err := c.fs.DeleteEntry(ctx, removedPath) + require.NoError(c.T(), err) + + filteredEntries := []testFileEntry{} + removed := 0 + for _, e := range c.contentMap { + if e.path[0] == removedPath[0] && e.path[1] == removedPath[1] { + continue + } + + filteredEntries = append(filteredEntries, e) + removed++ + } + c.contentMap = filteredEntries + require.NotZero(c.T(), removed) + + c.checkContentMap(c.T(), c.fs) + + err = c.fs.DeleteEntry(ctx, removedPath) + require.ErrorIs(c.T(), err, cinodefs.ErrEntryNotFound) + + c.checkContentMap(c.T(), c.fs) + + ep, err := c.fs.FindEntry(ctx, removedPath) + require.ErrorIs(c.T(), err, cinodefs.ErrEntryNotFound) + require.Nil(c.T(), ep) + + err = c.fs.DeleteEntry(ctx, []string{}) + require.ErrorIs(c.T(), err, cinodefs.ErrCantDeleteRoot) +} + +func (c *CinodeFSMultiFileTestSuite) TestDeleteTreatFileAsDirectory() { + ctx := context.Background() + + path := append(c.contentMap[0].path, "sub-file") + err := c.fs.DeleteEntry(ctx, path) + require.ErrorIs(c.T(), err, cinodefs.ErrNotADirectory) +} + +func (c *CinodeFSMultiFileTestSuite) TestResetDir() { + ctx := context.Background() + + removedPath := c.contentMap[0].path[:2] + + err := c.fs.ResetDir(ctx, removedPath) + require.NoError(c.T(), err) + + filteredEntries := []testFileEntry{} + removed := 0 + for _, e := range c.contentMap { + if e.path[0] == removedPath[0] && e.path[1] == removedPath[1] { + continue + } + + filteredEntries = append(filteredEntries, e) + removed++ + } + c.contentMap = filteredEntries + require.NotZero(c.T(), removed) + + c.checkContentMap(c.T(), c.fs) + + err = c.fs.ResetDir(ctx, removedPath) + require.NoError(c.T(), err) + + c.checkContentMap(c.T(), c.fs) + + ep, err := c.fs.FindEntry(ctx, removedPath) + require.ErrorIs(c.T(), err, cinodefs.ErrModifiedDirectory) + require.Nil(c.T(), ep) +} + +func (c *CinodeFSMultiFileTestSuite) TestSettingEntry() { + ctx := context.Background() + + c.T().Run("prevent treating file as directory", func(t *testing.T) { + path := append(c.contentMap[0].path, "sub-file") + _, err := c.fs.SetEntryFile(ctx, path, strings.NewReader("should not happen")) + require.ErrorIs(t, err, cinodefs.ErrNotADirectory) + }) + + c.T().Run("prevent setting empty path segment", func(t *testing.T) { + for _, path := range [][]string{ + {"", "subdir", "file.txt"}, + {"dir", "", "file.txt"}, + {"dir", "subdir", ""}, + } { + c.T().Run(strings.Join(path, "::"), func(t *testing.T) { + _, err := c.fs.SetEntryFile(ctx, path, strings.NewReader("should not succeed")) + require.ErrorIs(t, err, cinodefs.ErrEmptyName) + + }) + } + }) + + c.T().Run("tet root entrypoint on dirty filesystem", func(t *testing.T) { + ep1, err := c.fs.RootEntrypoint() + require.NoError(t, err) + + _, err = c.fs.SetEntryFile(ctx, c.contentMap[0].path, strings.NewReader("hello")) + require.NoError(t, err) + c.contentMap[0].content = "hello" + + ep2, err := c.fs.RootEntrypoint() + require.NoError(t, err) + + // Even though dirty, entrypoint won't change it's content + require.Equal(t, ep1.String(), ep2.String()) + + err = c.fs.Flush(ctx) + require.NoError(t, err) + + ep3, err := c.fs.RootEntrypoint() + require.NoError(t, err) + + require.Equal(t, ep1.String(), ep3.String()) + }) + + c.T().Run("test crete file entrypoint", func(t *testing.T) { + ep, err := c.fs.CreateFileEntrypoint(ctx, strings.NewReader("new file")) + require.NoError(t, err) + require.NotNil(t, ep) + + err = c.fs.SetEntry(context.Background(), []string{"new-file.txt"}, ep) + require.NoError(t, err) + + c.contentMap = append(c.contentMap, testFileEntry{ + path: []string{"new-file.txt"}, + content: "new file", + mimeType: ep.MimeType(), + }) + + c.checkContentMap(c.T(), c.fs) + }) +} + +func (c *CinodeFSMultiFileTestSuite) TestRootEPDirectoryOnDirtyFS() { + ctx := context.Background() + + rootDir, err := c.fs.FindEntry(ctx, []string{}) + require.NoError(c.T(), err) + + fs2, err := cinodefs.New(ctx, + blenc.FromDatastore(c.ds), + cinodefs.RootEntrypoint(rootDir), + ) + require.NoError(c.T(), err) + + ep1, err := fs2.RootEntrypoint() + require.NoError(c.T(), err) + require.Equal(c.T(), rootDir.String(), ep1.String()) + + _, err = fs2.SetEntryFile(ctx, c.contentMap[0].path, strings.NewReader("hello")) + require.NoError(c.T(), err) + + ep2, err := fs2.RootEntrypoint() + require.ErrorIs(c.T(), err, cinodefs.ErrModifiedDirectory) + require.Nil(c.T(), ep2) + + err = fs2.Flush(ctx) + require.NoError(c.T(), err) + + ep3, err := c.fs.RootEntrypoint() + require.NoError(c.T(), err) + + require.NotEqual(c.T(), ep1.String(), ep3.String()) +} + +func (c *CinodeFSMultiFileTestSuite) TestOpeningData() { + _, err := c.fs.OpenEntrypointData(context.Background(), nil) + require.ErrorIs(c.T(), err, cinodefs.ErrNilEntrypoint) + + _, err = c.fs.OpenEntryData(context.Background(), []string{"a", "b", "c"}) + require.ErrorIs(c.T(), err, cinodefs.ErrEntryNotFound) + + _, err = c.fs.OpenEntryData(context.Background(), []string{}) + require.ErrorIs(c.T(), err, cinodefs.ErrIsADirectory) + + contentReader, err := c.fs.OpenEntryData(context.Background(), c.contentMap[0].path) + require.NoError(c.T(), err) + content, err := io.ReadAll(contentReader) + require.NoError(c.T(), err) + require.Equal(c.T(), c.contentMap[0].content, string(content)) +} + +func (c *CinodeFSMultiFileTestSuite) TestSubLinksAndWriteOnlyPath() { + ctx := context.Background() + t := c.T() + path := append([]string{}, c.contentMap[0].path...) + path = append(path[:len(path)-1], "linked", "sub", "directory", "linked-file.txt") + linkPath := path[:len(path)-2] + + // Create normal file + ep, err := c.fs.SetEntryFile(ctx, path, strings.NewReader("linked-file")) + require.NoError(t, err) + c.contentMap = append(c.contentMap, testFileEntry{ + path: path, + content: "linked-file", + mimeType: ep.MimeType(), + }) + c.checkContentMap(t, c.fs) + + // Convert path to the file to a dynamic link + wi, err := c.fs.InjectDynamicLink(ctx, linkPath) + require.NoError(t, err) + require.NotNil(t, wi) + c.checkContentMap(t, c.fs) + + // Ensure flushing through the dynamic link works + err = c.fs.Flush(ctx) + require.NoError(t, err) + c.checkContentMap(t, c.fs) + + // Ensure the content can still be changed - corresponding auth info + // is still kept in the concept + _, err = c.fs.SetEntryFile(ctx, path, strings.NewReader("updated-linked-file")) + require.NoError(t, err) + c.contentMap[len(c.contentMap)-1].content = "updated-linked-file" + c.checkContentMap(t, c.fs) + + // Ensure flushing works after the change behind the link + err = c.fs.Flush(ctx) + require.NoError(t, err) + c.checkContentMap(t, c.fs) + + rootWriterInfo, err := c.fs.RootWriterInfo(ctx) + require.NoError(t, err) + + // Reopen the filesystem, but only with the root writer info + fs2, err := cinodefs.New(ctx, + blenc.FromDatastore(c.ds), + cinodefs.RootWriterInfoString(rootWriterInfo.String()), + ) + require.NoError(c.T(), err) + c.checkContentMap(c.T(), fs2) + + // Can not do any operation below the split point + ep, err = fs2.SetEntryFile(ctx, path, strings.NewReader("won't work")) + require.ErrorIs(t, err, cinodefs.ErrMissingWriterInfo) + require.Nil(t, ep) + + altPath := append(append([]string{}, path[:len(path)-1]...), "other", "directory", "path") + ep, err = fs2.SetEntryFile(ctx, altPath, strings.NewReader("won't work")) + require.ErrorIs(t, err, cinodefs.ErrMissingWriterInfo) + require.Nil(t, ep) + + err = fs2.ResetDir(ctx, path[:len(path)-1]) + require.ErrorIs(t, err, cinodefs.ErrMissingWriterInfo) + + err = fs2.DeleteEntry(ctx, path) + require.ErrorIs(t, err, cinodefs.ErrMissingWriterInfo) + + _, err = fs2.InjectDynamicLink(ctx, path) + require.ErrorIs(t, err, cinodefs.ErrMissingWriterInfo) +} + +func (c *CinodeFSMultiFileTestSuite) TestMaxLinksRedirects() { + t := c.T() + ctx := context.Background() + + entryPath := c.contentMap[0].path + linkPath := entryPath[:len(entryPath)-1] + + // Up to max links redirects, lookup must be allowed + for i := 0; i < c.maxLinkRedirects; i++ { + _, err := c.fs.InjectDynamicLink(ctx, linkPath) + require.NoError(t, err) + + _, err = c.fs.FindEntry(ctx, entryPath) + require.NoError(t, err) + } + + // Cross the max redirects count, next lookup should fail + _, err := c.fs.InjectDynamicLink(ctx, linkPath) + require.NoError(t, err) + + _, err = c.fs.FindEntry(ctx, entryPath) + require.ErrorIs(t, err, cinodefs.ErrTooManyRedirects) +} + +func (c *CinodeFSMultiFileTestSuite) TestExplicitMimeType() { + t := c.T() + ctx := context.Background() + entryPath := c.contentMap[0].path + const newMimeType = "forced-mime-type" + + _, err := c.fs.SetEntryFile(ctx, + entryPath, + strings.NewReader("modified content"), + cinodefs.SetMimeType(newMimeType), + ) + require.NoError(t, err) + + entry, err := c.fs.FindEntry(ctx, entryPath) + require.NoError(t, err) + require.Equal(t, newMimeType, entry.MimeType()) +} + +func (c *CinodeFSMultiFileTestSuite) TestExpiration() { + t := c.T() + ctx := context.Background() + entryPath := c.contentMap[0].path + + now := time.Now() + c.timeFunc = func() time.Time { return now } + + t.Run("not yet valid", func(t *testing.T) { + _, err := c.fs.SetEntryFile(ctx, + entryPath, + strings.NewReader("modified content"), + cinodefs.SetNotValidBefore(now.Add(time.Second)), + ) + require.NoError(t, err) + + _, err = c.fs.FindEntry(ctx, entryPath) + require.ErrorIs(t, err, cinodefs.ErrNotYetValid) + }) +} + +func TestFetchingWriterInfo(t *testing.T) { + t.Run("not a dynamic link", func(t *testing.T) { + fs, err := cinodefs.New( + context.Background(), + blenc.FromDatastore(datastore.InMemory()), + cinodefs.NewRootStaticDirectory(), + ) + require.NoError(t, err) + + wi, err := fs.RootWriterInfo(context.Background()) + require.ErrorIs(t, err, cinodefs.ErrModifiedDirectory) + require.Nil(t, wi) + + err = fs.Flush(context.Background()) + require.NoError(t, err) + + wi, err = fs.RootWriterInfo(context.Background()) + require.ErrorIs(t, err, cinodefs.ErrNotALink) + require.Nil(t, wi) + }) + + t.Run("dynamic link without writer info", func(t *testing.T) { + link, err := dynamiclink.Create(rand.Reader) + require.NoError(t, err) + ep := cinodefs.EntrypointFromBlobNameAndKey(link.BlobName(), link.EncryptionKey()) + + fs, err := cinodefs.New( + context.Background(), + blenc.FromDatastore(datastore.InMemory()), + // Set entrypoint without auth info + cinodefs.RootEntrypoint(ep), + ) + require.NoError(t, err) + + wi, err := fs.RootWriterInfo(context.Background()) + require.ErrorIs(t, err, cinodefs.ErrMissingWriterInfo) + require.Nil(t, wi) + }) +} diff --git a/pkg/cinodefs/cinodefs_options.go b/pkg/cinodefs/cinodefs_options.go index 9feefa8..79238fe 100644 --- a/pkg/cinodefs/cinodefs_options.go +++ b/pkg/cinodefs/cinodefs_options.go @@ -18,6 +18,8 @@ package cinodefs import ( "context" + "errors" + "fmt" "io" "time" @@ -28,10 +30,20 @@ const ( DefaultMaxLinksRedirects = 10 ) +var ( + ErrNegativeMaxLinksRedirects = errors.New("negative value of maximum links redirects") + ErrInvalidNilTimeFunc = errors.New("nil time function") + ErrInvalidNilRandSource = errors.New("nil random source") +) + type Option interface { apply(ctx context.Context, fs *cinodeFS) error } +type errOption struct{ err error } + +func (e errOption) apply(ctx context.Context, fs *cinodeFS) error { return e.err } + type optionFunc func(ctx context.Context, fs *cinodeFS) error func (f optionFunc) apply(ctx context.Context, fs *cinodeFS) error { @@ -39,6 +51,9 @@ func (f optionFunc) apply(ctx context.Context, fs *cinodeFS) error { } func MaxLinkRedirects(maxLinkRedirects int) Option { + if maxLinkRedirects < 0 { + return errOption{ErrNegativeMaxLinksRedirects} + } return optionFunc(func(ctx context.Context, fs *cinodeFS) error { fs.maxLinkRedirects = maxLinkRedirects return nil @@ -52,22 +67,28 @@ func RootEntrypoint(ep *Entrypoint) Option { }) } -func errOption(err error) Option { - return optionFunc(func(ctx context.Context, fs *cinodeFS) error { return err }) -} - func RootEntrypointString(eps string) Option { ep, err := EntrypointFromString(eps) if err != nil { - return errOption(err) + return errOption{err} } return RootEntrypoint(ep) } func RootWriterInfo(wi *WriterInfo) Option { + if wi == nil { + return errOption{fmt.Errorf( + "%w: nil", + ErrInvalidWriterInfoData, + )} + } bn, err := common.BlobNameFromBytes(wi.wi.BlobName) if err != nil { - return errOption(err) + return errOption{fmt.Errorf( + "%w: %w", + ErrInvalidWriterInfoData, + err, + )} } key := common.BlobKeyFromBytes(wi.wi.Key) @@ -83,13 +104,16 @@ func RootWriterInfo(wi *WriterInfo) Option { func RootWriterInfoString(wis string) Option { wi, err := WriterInfoFromString(wis) if err != nil { - return errOption(err) + return errOption{err} } return RootWriterInfo(wi) } func TimeFunc(f func() time.Time) Option { + if f == nil { + return errOption{ErrInvalidNilTimeFunc} + } return optionFunc(func(ctx context.Context, fs *cinodeFS) error { fs.timeFunc = f return nil @@ -97,6 +121,9 @@ func TimeFunc(f func() time.Time) Option { } func RandSource(r io.Reader) Option { + if r == nil { + return errOption{ErrInvalidNilRandSource} + } return optionFunc(func(ctx context.Context, fs *cinodeFS) error { fs.randSource = r return nil @@ -107,7 +134,7 @@ func RandSource(r io.Reader) Option { // dynamic link as the root func NewRootDynamicLink() Option { return optionFunc(func(ctx context.Context, fs *cinodeFS) error { - newLinkEntrypoint, err := fs.GenerateNewDynamicLinkEntrypoint() + newLinkEntrypoint, _, err := fs.generateNewDynamicLinkEntrypoint() if err != nil { return err } diff --git a/pkg/cinodefs/cinodefs_options_bb_test.go b/pkg/cinodefs/cinodefs_options_bb_test.go new file mode 100644 index 0000000..6baa2e8 --- /dev/null +++ b/pkg/cinodefs/cinodefs_options_bb_test.go @@ -0,0 +1,115 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cinodefs_test + +import ( + "context" + "errors" + "testing" + "testing/iotest" + + "github.com/cinode/go/pkg/blenc" + "github.com/cinode/go/pkg/cinodefs" + "github.com/cinode/go/pkg/datastore" + "github.com/stretchr/testify/require" +) + +func TestInvalidCinodeFSOptions(t *testing.T) { + t.Run("no blenc", func(t *testing.T) { + cfs, err := cinodefs.New(context.Background(), nil) + require.ErrorIs(t, err, cinodefs.ErrInvalidBE) + require.Nil(t, cfs) + }) + + be := blenc.FromDatastore(datastore.InMemory()) + + t.Run("no root info", func(t *testing.T) { + cfs, err := cinodefs.New(context.Background(), be) + require.ErrorIs(t, err, cinodefs.ErrMissingRootInfo) + require.Nil(t, cfs) + }) + + t.Run("negative max links redirects", func(t *testing.T) { + cfs, err := cinodefs.New(context.Background(), be, + cinodefs.NewRootStaticDirectory(), + cinodefs.MaxLinkRedirects(-1), + ) + require.ErrorIs(t, err, cinodefs.ErrNegativeMaxLinksRedirects) + require.Nil(t, cfs) + }) + + t.Run("invalid entrypoint string", func(t *testing.T) { + cfs, err := cinodefs.New(context.Background(), be, + cinodefs.RootEntrypointString(""), + ) + require.ErrorIs(t, err, cinodefs.ErrInvalidEntrypointData) + require.Nil(t, cfs) + }) + + t.Run("invalid writer info string", func(t *testing.T) { + cfs, err := cinodefs.New(context.Background(), be, + cinodefs.RootWriterInfoString(""), + ) + require.ErrorIs(t, err, cinodefs.ErrInvalidWriterInfoData) + require.Nil(t, cfs) + }) + + t.Run("invalid nil writer info", func(t *testing.T) { + cfs, err := cinodefs.New(context.Background(), be, + cinodefs.RootWriterInfo(nil), + ) + require.ErrorIs(t, err, cinodefs.ErrInvalidWriterInfoData) + require.Nil(t, cfs) + }) + + t.Run("invalid writer info", func(t *testing.T) { + cfs, err := cinodefs.New(context.Background(), be, + cinodefs.RootWriterInfo(&cinodefs.WriterInfo{}), + ) + require.ErrorIs(t, err, cinodefs.ErrInvalidWriterInfoData) + require.Nil(t, cfs) + }) + + t.Run("invalid time func", func(t *testing.T) { + cfs, err := cinodefs.New(context.Background(), be, + cinodefs.TimeFunc(nil), + ) + require.ErrorIs(t, err, cinodefs.ErrInvalidNilTimeFunc) + require.Nil(t, cfs) + }) + + t.Run("invalid nil random source", func(t *testing.T) { + cfs, err := cinodefs.New(context.Background(), be, + cinodefs.RandSource(nil), + ) + require.ErrorIs(t, err, cinodefs.ErrInvalidNilRandSource) + require.Nil(t, cfs) + }) + + t.Run("invalid random source", func(t *testing.T) { + // Error will manifest itself while random data source + // is needed which only takes place when new random + // dynamic link is requested + injectedErr := errors.New("random source error") + cfs, err := cinodefs.New(context.Background(), be, + cinodefs.RandSource(iotest.ErrReader(injectedErr)), + cinodefs.NewRootDynamicLink(), + ) + require.ErrorIs(t, err, injectedErr) + require.Nil(t, cfs) + }) +} diff --git a/pkg/cinodefs/cinodefs_traverse.go b/pkg/cinodefs/cinodefs_traverse.go index 0f8fb24..9c54074 100644 --- a/pkg/cinodefs/cinodefs_traverse.go +++ b/pkg/cinodefs/cinodefs_traverse.go @@ -31,9 +31,9 @@ type traverseGoalFunc func( ) type traverseOptions struct { - createNodes bool - doNotCache bool - maxLinkDepth int + createNodes bool + doNotCache bool + maxLinkRedirects int } // Generic graph traversal function, it follows given path, once the endpoint @@ -50,6 +50,8 @@ func (fs *cinodeFS) traverseGraph( } } + opts.maxLinkRedirects = fs.maxLinkRedirects + changedEntrypoint, _, err := fs.rootEP.traverse( ctx, // context &fs.c, // graph context diff --git a/pkg/cinodefs/context.go b/pkg/cinodefs/context.go index 9e543e4..99576ee 100644 --- a/pkg/cinodefs/context.go +++ b/pkg/cinodefs/context.go @@ -115,13 +115,13 @@ func (c *graphContext) createProtobufMessage( return nil, fmt.Errorf("serialization failed: %w", err) } - bn, key, wi, err := c.be.Create(ctx, blobType, bytes.NewReader(data)) + bn, key, ai, err := c.be.Create(ctx, blobType, bytes.NewReader(data)) if err != nil { return nil, fmt.Errorf("write failed: %w", err) } - if wi != nil { - c.authInfos[bn.String()] = wi + if ai != nil { + c.authInfos[bn.String()] = ai } return &Entrypoint{ diff --git a/pkg/cinodefs/entrypoint_bb_test.go b/pkg/cinodefs/entrypoint_bb_test.go new file mode 100644 index 0000000..cebe0e0 --- /dev/null +++ b/pkg/cinodefs/entrypoint_bb_test.go @@ -0,0 +1,77 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cinodefs_test + +import ( + "testing" + + "github.com/cinode/go/pkg/cinodefs" + "github.com/cinode/go/pkg/cinodefs/internal/protobuf" + "github.com/cinode/go/testvectors/testblobs" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func TestEntrypointFromStringFailures(t *testing.T) { + for _, d := range []struct { + s string + errContains string + }{ + {"", "empty string"}, + {"not-a-base64-string!!!!!!!!", "not a base58 string"}, + {"aaaaaaaa", "protobuf parse error"}, + } { + t.Run(d.s, func(t *testing.T) { + wi, err := cinodefs.EntrypointFromString(d.s) + require.ErrorIs(t, err, cinodefs.ErrInvalidEntrypointData) + require.ErrorContains(t, err, d.errContains) + require.Nil(t, wi) + }) + } +} + +func TestInvalidEntrypointData(t *testing.T) { + for _, d := range []struct { + n string + p *protobuf.Entrypoint + errContains string + }{ + { + "invalid blob name", + &protobuf.Entrypoint{}, + "invalid blob name", + }, + { + "mime type set for link", + &protobuf.Entrypoint{ + BlobName: testblobs.DynamicLink.BlobName.Bytes(), + MimeType: "test-mimetype", + }, + "link can not have mimetype set", + }, + } { + t.Run(d.n, func(t *testing.T) { + bytes, err := proto.Marshal(d.p) + require.NoError(t, err) + + ep, err := cinodefs.EntrypointFromBytes(bytes) + require.ErrorIs(t, err, cinodefs.ErrInvalidEntrypointData) + require.ErrorContains(t, err, d.errContains) + require.Nil(t, ep) + }) + } +} diff --git a/pkg/cinodefs/entrypoint_options.go b/pkg/cinodefs/entrypoint_options.go index 57505c1..8699bb3 100644 --- a/pkg/cinodefs/entrypoint_options.go +++ b/pkg/cinodefs/entrypoint_options.go @@ -22,15 +22,12 @@ import ( ) type EntrypointOption interface { - apply(ctx context.Context, ep *Entrypoint) error + apply(ctx context.Context, ep *Entrypoint) } type entrypointOptionBasicFunc func(ep *Entrypoint) -func (f entrypointOptionBasicFunc) apply(ctx context.Context, ep *Entrypoint) error { - f(ep) - return nil -} +func (f entrypointOptionBasicFunc) apply(ctx context.Context, ep *Entrypoint) { f(ep) } func SetMimeType(mimeType string) EntrypointOption { return entrypointOptionBasicFunc(func(ep *Entrypoint) { @@ -50,12 +47,10 @@ func SetNotValidAfter(t time.Time) EntrypointOption { }) } -func entrypointFromOptions(ctx context.Context, opts ...EntrypointOption) (*Entrypoint, error) { +func entrypointFromOptions(ctx context.Context, opts ...EntrypointOption) *Entrypoint { ep := &Entrypoint{} for _, o := range opts { - if err := o.apply(ctx, ep); err != nil { - return nil, err - } + o.apply(ctx, ep) } - return ep, nil + return ep } diff --git a/pkg/cinodefs/node_link.go b/pkg/cinodefs/node_link.go index ede746a..140228b 100644 --- a/pkg/cinodefs/node_link.go +++ b/pkg/cinodefs/node_link.go @@ -73,7 +73,7 @@ func (c *nodeLink) traverse( dirtyState, error, ) { - if linkDepth > opts.maxLinkDepth { + if linkDepth >= opts.maxLinkRedirects { return nil, 0, ErrTooManyRedirects } diff --git a/pkg/cinodefs/writerinfo_bb_test.go b/pkg/cinodefs/writerinfo_bb_test.go new file mode 100644 index 0000000..72d2b34 --- /dev/null +++ b/pkg/cinodefs/writerinfo_bb_test.go @@ -0,0 +1,42 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cinodefs_test + +import ( + "testing" + + "github.com/cinode/go/pkg/cinodefs" + "github.com/stretchr/testify/require" +) + +func TestWriterInfoFromStringFailures(t *testing.T) { + for _, d := range []struct { + s string + errContains string + }{ + {"", "empty string"}, + {"not-a-base64-string!!!!!!!!", "not a base58 string"}, + {"aaaaaaaa", "protobuf parse error"}, + } { + t.Run(d.s, func(t *testing.T) { + wi, err := cinodefs.WriterInfoFromString(d.s) + require.ErrorIs(t, err, cinodefs.ErrInvalidWriterInfoData) + require.ErrorContains(t, err, d.errContains) + require.Nil(t, wi) + }) + } +} From 768886f8164971c1d92fd43e758053a3d483cffc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Wed, 1 Nov 2023 22:35:42 +0100 Subject: [PATCH 24/29] Remove expiration support for now. It is a larger topic and will be handled separately. --- pkg/cinodefs/cinodefs_interface_bb_test.go | 21 --------------------- pkg/cinodefs/entrypoint.go | 16 ---------------- pkg/cinodefs/entrypoint_options.go | 13 ------------- 3 files changed, 50 deletions(-) diff --git a/pkg/cinodefs/cinodefs_interface_bb_test.go b/pkg/cinodefs/cinodefs_interface_bb_test.go index 1f56727..cd1b0ea 100644 --- a/pkg/cinodefs/cinodefs_interface_bb_test.go +++ b/pkg/cinodefs/cinodefs_interface_bb_test.go @@ -541,27 +541,6 @@ func (c *CinodeFSMultiFileTestSuite) TestExplicitMimeType() { require.Equal(t, newMimeType, entry.MimeType()) } -func (c *CinodeFSMultiFileTestSuite) TestExpiration() { - t := c.T() - ctx := context.Background() - entryPath := c.contentMap[0].path - - now := time.Now() - c.timeFunc = func() time.Time { return now } - - t.Run("not yet valid", func(t *testing.T) { - _, err := c.fs.SetEntryFile(ctx, - entryPath, - strings.NewReader("modified content"), - cinodefs.SetNotValidBefore(now.Add(time.Second)), - ) - require.NoError(t, err) - - _, err = c.fs.FindEntry(ctx, entryPath) - require.ErrorIs(t, err, cinodefs.ErrNotYetValid) - }) -} - func TestFetchingWriterInfo(t *testing.T) { t.Run("not a dynamic link", func(t *testing.T) { fs, err := cinodefs.New( diff --git a/pkg/cinodefs/entrypoint.go b/pkg/cinodefs/entrypoint.go index c8dcfcc..1359543 100644 --- a/pkg/cinodefs/entrypoint.go +++ b/pkg/cinodefs/entrypoint.go @@ -19,7 +19,6 @@ package cinodefs import ( "errors" "fmt" - "time" "github.com/cinode/go/pkg/blobtypes" "github.com/cinode/go/pkg/cinodefs/internal/protobuf" @@ -137,18 +136,3 @@ func (e *Entrypoint) IsDir() bool { func (e *Entrypoint) MimeType() string { return e.ep.MimeType } - -func (e *Entrypoint) IsValid(now time.Time) error { - nowMicro := now.UnixMicro() - if e.ep.NotValidBeforeUnixMicro != 0 { - if e.ep.NotValidBeforeUnixMicro > nowMicro { - return ErrNotYetValid - } - } - if e.ep.NotValidAfterUnixMicro != 0 { - if e.ep.NotValidAfterUnixMicro < nowMicro { - return ErrExpired - } - } - return nil -} diff --git a/pkg/cinodefs/entrypoint_options.go b/pkg/cinodefs/entrypoint_options.go index 8699bb3..be6d24f 100644 --- a/pkg/cinodefs/entrypoint_options.go +++ b/pkg/cinodefs/entrypoint_options.go @@ -18,7 +18,6 @@ package cinodefs import ( "context" - "time" ) type EntrypointOption interface { @@ -35,18 +34,6 @@ func SetMimeType(mimeType string) EntrypointOption { }) } -func SetNotValidBefore(t time.Time) EntrypointOption { - return entrypointOptionBasicFunc(func(ep *Entrypoint) { - ep.ep.NotValidBeforeUnixMicro = t.UnixMicro() - }) -} - -func SetNotValidAfter(t time.Time) EntrypointOption { - return entrypointOptionBasicFunc(func(ep *Entrypoint) { - ep.ep.NotValidAfterUnixMicro = t.UnixMicro() - }) -} - func entrypointFromOptions(ctx context.Context, opts ...EntrypointOption) *Entrypoint { ep := &Entrypoint{} for _, o := range opts { From 55132bfaf858e8e2de4e8bef41587ff51829fd6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Mon, 6 Nov 2023 12:27:15 +0100 Subject: [PATCH 25/29] Improve test coverage of cinodefs + small fixes --- pkg/cinodefs/cinodefs_interface.go | 34 ++- pkg/cinodefs/cinodefs_interface_bb_test.go | 328 ++++++++++++++++++++- pkg/cinodefs/entrypoint.go | 2 +- pkg/cinodefs/node_unloaded.go | 12 +- pkg/common/auth_info.go | 2 +- 5 files changed, 352 insertions(+), 26 deletions(-) diff --git a/pkg/cinodefs/cinodefs_interface.go b/pkg/cinodefs/cinodefs_interface.go index bb4f504..9d03ed8 100644 --- a/pkg/cinodefs/cinodefs_interface.go +++ b/pkg/cinodefs/cinodefs_interface.go @@ -20,6 +20,7 @@ import ( "context" "crypto/rand" "errors" + "fmt" "io" "mime" "net/http" @@ -35,22 +36,23 @@ import ( ) var ( - ErrInvalidBE = errors.New("invalid BE argument") - ErrCantOpenDir = errors.New("can not open directory") - ErrTooManyRedirects = errors.New("too many link redirects") - ErrCantComputeBlobKey = errors.New("can not compute blob keys") - ErrModifiedDirectory = errors.New("can not get entrypoint for a directory, unsaved content") - ErrCantDeleteRoot = errors.New("can not delete root object") - ErrNotADirectory = errors.New("entry is not a directory") - ErrNotALink = errors.New("entry is not a link") - ErrNilEntrypoint = errors.New("nil entrypoint") - ErrEmptyName = errors.New("entry name can not be empty") - ErrDuplicateEntry = errors.New("duplicate entry") - ErrEntryNotFound = errors.New("entry not found") - ErrIsADirectory = errors.New("entry is a directory") - ErrInvalidDirectoryData = errors.New("invalid directory data") - ErrCantWriteDirectory = errors.New("can not write directory") - ErrMissingRootInfo = errors.New("root info not specified") + ErrInvalidBE = errors.New("invalid BE argument") + ErrCantOpenDir = errors.New("can not open directory") + ErrCantOpenDirDuplicateEntry = fmt.Errorf("%w: duplicate entry", ErrCantOpenDir) + ErrCantOpenLink = errors.New("can not open link") + ErrTooManyRedirects = errors.New("too many link redirects") + ErrCantComputeBlobKey = errors.New("can not compute blob keys") + ErrModifiedDirectory = errors.New("can not get entrypoint for a directory, unsaved content") + ErrCantDeleteRoot = errors.New("can not delete root object") + ErrNotADirectory = errors.New("entry is not a directory") + ErrNotALink = errors.New("entry is not a link") + ErrNilEntrypoint = errors.New("nil entrypoint") + ErrEmptyName = errors.New("entry name can not be empty") + ErrEntryNotFound = errors.New("entry not found") + ErrIsADirectory = errors.New("entry is a directory") + ErrInvalidDirectoryData = errors.New("invalid directory data") + ErrCantWriteDirectory = errors.New("can not write directory") + ErrMissingRootInfo = errors.New("root info not specified") ) const ( diff --git a/pkg/cinodefs/cinodefs_interface_bb_test.go b/pkg/cinodefs/cinodefs_interface_bb_test.go index cd1b0ea..5f3d936 100644 --- a/pkg/cinodefs/cinodefs_interface_bb_test.go +++ b/pkg/cinodefs/cinodefs_interface_bb_test.go @@ -17,20 +17,27 @@ limitations under the License. package cinodefs_test import ( + "bytes" "context" "crypto/rand" + "errors" "fmt" "io" "strings" "testing" + "testing/iotest" "time" "github.com/cinode/go/pkg/blenc" "github.com/cinode/go/pkg/cinodefs" + "github.com/cinode/go/pkg/cinodefs/internal/protobuf" + "github.com/cinode/go/pkg/common" "github.com/cinode/go/pkg/datastore" "github.com/cinode/go/pkg/internal/blobtypes/dynamiclink" + "github.com/cinode/go/pkg/utilities/golang" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "google.golang.org/protobuf/proto" ) func TestCinodeFSSingleFileScenario(t *testing.T) { @@ -73,6 +80,38 @@ func TestCinodeFSSingleFileScenario(t *testing.T) { } } +type testBEWrapper struct { + blenc.BE + + createFunc func( + ctx context.Context, blobType common.BlobType, r io.Reader, + ) (*common.BlobName, *common.BlobKey, *common.AuthInfo, error) + + updateFunc func( + ctx context.Context, name *common.BlobName, ai *common.AuthInfo, + key *common.BlobKey, r io.Reader, + ) error +} + +func (w *testBEWrapper) Create( + ctx context.Context, blobType common.BlobType, r io.Reader, +) (*common.BlobName, *common.BlobKey, *common.AuthInfo, error) { + if w.createFunc != nil { + return w.createFunc(ctx, blobType, r) + } + return w.BE.Create(ctx, blobType, r) +} + +func (w *testBEWrapper) Update( + ctx context.Context, name *common.BlobName, ai *common.AuthInfo, + key *common.BlobKey, r io.Reader, +) error { + if w.updateFunc != nil { + return w.updateFunc(ctx, name, ai, key, r) + } + return w.BE.Update(ctx, name, ai, key, r) +} + type testFileEntry struct { path []string content string @@ -83,12 +122,20 @@ type CinodeFSMultiFileTestSuite struct { suite.Suite ds datastore.DS + be testBEWrapper fs cinodefs.FS contentMap []testFileEntry maxLinkRedirects int + randSource io.Reader timeFunc func() time.Time } +type randReaderForCinodeFSMultiFileTestSuite CinodeFSMultiFileTestSuite + +func (r *randReaderForCinodeFSMultiFileTestSuite) Read(b []byte) (int, error) { + return r.randSource.Read(b) +} + func TestCinodeFSMultiFileTestSuite(t *testing.T) { suite.Run(t, &CinodeFSMultiFileTestSuite{ maxLinkRedirects: 5, @@ -99,12 +146,17 @@ func (c *CinodeFSMultiFileTestSuite) SetupTest() { ctx := context.Background() c.timeFunc = time.Now + c.randSource = rand.Reader c.ds = datastore.InMemory() + c.be = testBEWrapper{ + BE: blenc.FromDatastore(c.ds), + } fs, err := cinodefs.New(ctx, - blenc.FromDatastore(c.ds), + &c.be, cinodefs.NewRootDynamicLink(), cinodefs.MaxLinkRedirects(c.maxLinkRedirects), - cinodefs.TimeFunc(c.timeFunc), + cinodefs.TimeFunc(func() time.Time { return c.timeFunc() }), + cinodefs.RandSource((*randReaderForCinodeFSMultiFileTestSuite)(c)), ) require.NoError(c.T(), err) require.NotNil(c.T(), fs) @@ -541,6 +593,278 @@ func (c *CinodeFSMultiFileTestSuite) TestExplicitMimeType() { require.Equal(t, newMimeType, entry.MimeType()) } +func (c *CinodeFSMultiFileTestSuite) TestMalformedDirectory() { + var ep protobuf.Entrypoint + err := proto.Unmarshal( + golang.Must(c.fs.FindEntry(context.Background(), c.contentMap[0].path)).Bytes(), + &ep, + ) + require.NoError(c.T(), err) + + var brokenEP protobuf.Entrypoint + proto.Merge(&brokenEP, &ep) + brokenEP.BlobName = []byte{} + + for _, d := range []struct { + n string + d []byte + err error + }{ + { + "malformed data", + []byte{23, 45, 67, 89, 12, 34, 56, 78, 90}, // Some malformed message + cinodefs.ErrCantOpenDir, + }, + { + "entry with empty name", + golang.Must(proto.Marshal(&protobuf.Directory{ + Entries: []*protobuf.Directory_Entry{{ + Name: "", + }}, + })), + cinodefs.ErrEmptyName, + }, + { + "two entries with the same name", + golang.Must(proto.Marshal(&protobuf.Directory{ + Entries: []*protobuf.Directory_Entry{ + {Name: "entry", Ep: &ep}, + {Name: "entry", Ep: &ep}, + }, + })), + cinodefs.ErrCantOpenDirDuplicateEntry, + }, + { + "missing entrypoint", + golang.Must(proto.Marshal(&protobuf.Directory{ + Entries: []*protobuf.Directory_Entry{ + {Name: "entry"}, + }, + })), + cinodefs.ErrInvalidEntrypointDataNil, + }, + { + "missing blob name", + golang.Must(proto.Marshal(&protobuf.Directory{ + Entries: []*protobuf.Directory_Entry{ + {Name: "entry", Ep: &brokenEP}, + }, + })), + common.ErrInvalidBlobName, + }, + } { + c.T().Run(d.n, func(t *testing.T) { + _, err := c.fs.SetEntryFile(context.Background(), + []string{"dir"}, + bytes.NewReader(d.d), + cinodefs.SetMimeType(cinodefs.CinodeDirMimeType), + ) + require.NoError(t, err) + + _, err = c.fs.FindEntry(context.Background(), []string{"dir", "entry"}) + require.ErrorIs(t, err, cinodefs.ErrCantOpenDir) + require.ErrorIs(t, err, d.err) + + // TODO: We should be able to set new entry even if the underlying object is broken + err = c.fs.DeleteEntry(context.Background(), []string{"dir"}) + require.NoError(t, err) + }) + } +} + +func (c *CinodeFSMultiFileTestSuite) TestMalformedLink() { + var ep protobuf.Entrypoint + err := proto.Unmarshal( + golang.Must(c.fs.FindEntry(context.Background(), c.contentMap[0].path)).Bytes(), + &ep, + ) + require.NoError(c.T(), err) + + var brokenEP protobuf.Entrypoint + proto.Merge(&brokenEP, &ep) + brokenEP.BlobName = []byte{} + + _, err = c.fs.SetEntryFile(context.Background(), []string{"link", "file"}, strings.NewReader("test")) + require.NoError(c.T(), err) + + linkWI_, err := c.fs.InjectDynamicLink(context.Background(), []string{"link"}) + require.NoError(c.T(), err) + + // Flush is needed so that we can update entrypoint data and the fs cache won't get into our way + err = c.fs.Flush(context.Background()) + require.NoError(c.T(), err) + + for _, d := range []struct { + n string + d []byte + err error + }{ + { + "malformed data", + []byte{23, 45, 67, 89, 12, 34, 56, 78, 90}, // Some malformed message + cinodefs.ErrCantOpenLink, + }, + { + "missing target blob name", + golang.Must(proto.Marshal(&brokenEP)), + common.ErrInvalidBlobName, + }, + } { + c.T().Run(d.n, func(t *testing.T) { + var linkWI protobuf.WriterInfo + err = proto.Unmarshal(linkWI_.Bytes(), &linkWI) + require.NoError(c.T(), err) + linkBlobName := golang.Must(common.BlobNameFromBytes(linkWI.BlobName)) + linkAuthInfo := common.AuthInfoFromBytes(linkWI.AuthInfo) + linkKey := common.BlobKeyFromBytes(linkWI.Key) + + err = c.be.Update(context.Background(), + linkBlobName, linkAuthInfo, linkKey, bytes.NewReader(d.d), + ) + require.NoError(t, err) + + _, err = c.fs.FindEntry(context.Background(), []string{"link", "file"}) + require.ErrorIs(t, err, cinodefs.ErrCantOpenLink) + require.ErrorIs(t, err, d.err) + }) + } +} + +func (c *CinodeFSMultiFileTestSuite) TestPathWithMultipleLinks() { + path := []string{ + "multi", + "level", + "path", + "with", + "more", + "than", + "one", + "link", + } + ctx := context.Background() + t := c.T() + + // Create test entry + const initialContent = "initial content" + ep, err := c.fs.SetEntryFile(ctx, path, strings.NewReader(initialContent)) + require.NoError(t, err) + + // Inject few links among the path to the entry + for _, splitPoint := range []int{2, 6, 4} { + _, err = c.fs.InjectDynamicLink(ctx, path[:splitPoint]) + require.NoError(t, err) + + err = c.fs.Flush(ctx) + require.NoError(t, err) + } + + // Create parallel filesystem + rootEP, err := c.fs.RootEntrypoint() + require.NoError(t, err) + + fs2, err := cinodefs.New(ctx, + blenc.FromDatastore(c.ds), + cinodefs.RootEntrypointString(rootEP.String()), + ) + require.NoError(t, err) + + c.contentMap = append(c.contentMap, testFileEntry{ + path: path, + content: initialContent, + mimeType: ep.MimeType(), + }) + c.checkContentMap(t, c.fs) + + // Modify the content of the file in the original filesystem, not yet flushed + const modifiedContent1 = "modified content 1" + _, err = c.fs.SetEntryFile(ctx, path, strings.NewReader(modifiedContent1)) + require.NoError(t, err) + + // Change not yet observed through the second filesystem due to no flush + c.checkContentMap(t, fs2) + + err = c.fs.Flush(ctx) + require.NoError(t, err) + + // Change must now be observed through the second filesystem + c.contentMap[len(c.contentMap)-1].content = modifiedContent1 + c.checkContentMap(t, c.fs) + c.checkContentMap(t, fs2) +} + +func (c *CinodeFSMultiFileTestSuite) TestBlobWriteErrorWhenCreatingFile() { + injectedErr := errors.New("entry file create error") + c.be.createFunc = func(ctx context.Context, blobType common.BlobType, r io.Reader, + ) (*common.BlobName, *common.BlobKey, *common.AuthInfo, error) { + return nil, nil, nil, injectedErr + } + + _, err := c.fs.SetEntryFile(context.Background(), []string{"file"}, strings.NewReader("test")) + require.ErrorIs(c.T(), err, injectedErr) +} + +func (c *CinodeFSMultiFileTestSuite) TestBlobWriteErrorWhenFlushing() { + _, err := c.fs.SetEntryFile(context.Background(), []string{"file"}, strings.NewReader("test")) + require.NoError(c.T(), err) + + injectedErr := errors.New("flush error") + c.be.createFunc = func(ctx context.Context, blobType common.BlobType, r io.Reader, + ) (*common.BlobName, *common.BlobKey, *common.AuthInfo, error) { + return nil, nil, nil, injectedErr + } + + err = c.fs.Flush(context.Background()) + require.ErrorIs(c.T(), err, injectedErr) +} + +func (c *CinodeFSMultiFileTestSuite) TestLinkGenerationError() { + injectedErr := errors.New("rand data read error") + + c.randSource = iotest.ErrReader(injectedErr) + + _, err := c.fs.InjectDynamicLink( + context.Background(), + c.contentMap[0].path[:2], + ) + require.ErrorIs(c.T(), err, injectedErr) +} + +func (c *CinodeFSMultiFileTestSuite) TestBlobWriteWhenCreatingLink() { + injectedErr := errors.New("link creation error") + c.be.updateFunc = func(ctx context.Context, name *common.BlobName, ai *common.AuthInfo, key *common.BlobKey, r io.Reader) error { + return injectedErr + } + + _, err := c.fs.InjectDynamicLink(context.Background(), c.contentMap[0].path[:2]) + require.NoError(c.T(), err) + + err = c.fs.Flush(context.Background()) + require.ErrorIs(c.T(), err, injectedErr) +} + +func (c *CinodeFSMultiFileTestSuite) TestReadFailureMissingKey() { + var epProto protobuf.Entrypoint + err := proto.Unmarshal( + golang.Must(c.fs.FindEntry(context.Background(), c.contentMap[0].path)).Bytes(), + &epProto, + ) + require.NoError(c.T(), err) + + // Generate derived EP without key + epProto.KeyInfo.Key = nil + ep := golang.Must(cinodefs.EntrypointFromBytes( + golang.Must(proto.Marshal(&epProto)), + )) + + // Replace current entrypoint with one without the key + err = c.fs.SetEntry(context.Background(), c.contentMap[0].path, ep) + require.NoError(c.T(), err) + + r, err := c.fs.OpenEntryData(context.Background(), c.contentMap[0].path) + require.ErrorIs(c.T(), err, cinodefs.ErrMissingKeyInfo) + require.Nil(c.T(), r) +} + func TestFetchingWriterInfo(t *testing.T) { t.Run("not a dynamic link", func(t *testing.T) { fs, err := cinodefs.New( diff --git a/pkg/cinodefs/entrypoint.go b/pkg/cinodefs/entrypoint.go index 1359543..adae7cb 100644 --- a/pkg/cinodefs/entrypoint.go +++ b/pkg/cinodefs/entrypoint.go @@ -90,7 +90,7 @@ func expandEntrypointProto(ep *Entrypoint) error { // Extract blob name from entrypoint bn, err := common.BlobNameFromBytes(ep.ep.BlobName) if err != nil { - return fmt.Errorf("%w: %s", ErrInvalidEntrypointData, err) + return fmt.Errorf("%w: %w", ErrInvalidEntrypointData, err) } ep.bn = bn diff --git a/pkg/cinodefs/node_unloaded.go b/pkg/cinodefs/node_unloaded.go index 0d5fe3b..cd39294 100644 --- a/pkg/cinodefs/node_unloaded.go +++ b/pkg/cinodefs/node_unloaded.go @@ -83,12 +83,12 @@ func (c *nodeUnloaded) loadEntrypointLink(ctx context.Context, gc *graphContext) targetEP := &Entrypoint{} err := gc.readProtobufMessage(ctx, c.ep, &targetEP.ep) if err != nil { - return nil, err + return nil, fmt.Errorf("%w: %w", ErrCantOpenLink, err) } err = expandEntrypointProto(targetEP) if err != nil { - return nil, err + return nil, fmt.Errorf("%w: %w", ErrCantOpenLink, err) } return &nodeLink{ @@ -102,22 +102,22 @@ func (c *nodeUnloaded) loadEntrypointDir(ctx context.Context, gc *graphContext) msg := &protobuf.Directory{} err := gc.readProtobufMessage(ctx, c.ep, msg) if err != nil { - return nil, err + return nil, fmt.Errorf("%w: %w", ErrCantOpenDir, err) } dir := make(map[string]node, len(msg.Entries)) for _, entry := range msg.Entries { if entry.Name == "" { - return nil, ErrEmptyName + return nil, fmt.Errorf("%w: %w", ErrCantOpenDir, ErrEmptyName) } if _, exists := dir[entry.Name]; exists { - return nil, fmt.Errorf("%w: %s", ErrDuplicateEntry, entry.Name) + return nil, fmt.Errorf("%w: %s", ErrCantOpenDirDuplicateEntry, entry.Name) } ep, err := entrypointFromProtobuf(entry.Ep) if err != nil { - return nil, err + return nil, fmt.Errorf("%w: %w", ErrCantOpenDir, err) } dir[entry.Name] = &nodeUnloaded{ep: ep} diff --git a/pkg/common/auth_info.go b/pkg/common/auth_info.go index f926d7c..6891bc3 100644 --- a/pkg/common/auth_info.go +++ b/pkg/common/auth_info.go @@ -24,6 +24,6 @@ import "crypto/subtle" // to update the content of the blob. The representation is specific to the blob type type AuthInfo struct{ data []byte } -func AuthInfoFromBytes(iv []byte) *AuthInfo { return &AuthInfo{data: copyBytes(iv)} } +func AuthInfoFromBytes(ai []byte) *AuthInfo { return &AuthInfo{data: copyBytes(ai)} } func (a *AuthInfo) Bytes() []byte { return copyBytes(a.data) } func (a *AuthInfo) Equal(a2 *AuthInfo) bool { return subtle.ConstantTimeCompare(a.data, a2.data) == 1 } From 4a98d10f5ab99f6a3ee9390c212430aa5e489523 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Sun, 19 Nov 2023 18:26:03 +0100 Subject: [PATCH 26/29] Better static_datastore cmd test --- .vscode/settings.json | 1 + cmd/static_datastore_builder/main.go | 13 +- pkg/cmd/static_datastore/compile.go | 22 +- pkg/cmd/static_datastore/root.go | 12 +- .../static_datastore/static_datastore_test.go | 240 ++++++++++++++---- 5 files changed, 217 insertions(+), 71 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index df3a0f0..343ffc3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -19,6 +19,7 @@ "fsys", "goveralls", "Hasher", + "homefile", "jbenet", "protobuf", "securefifo", diff --git a/cmd/static_datastore_builder/main.go b/cmd/static_datastore_builder/main.go index 121d74e..639ece2 100644 --- a/cmd/static_datastore_builder/main.go +++ b/cmd/static_datastore_builder/main.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,8 +16,15 @@ limitations under the License. package main -import "github.com/cinode/go/pkg/cmd/static_datastore" +import ( + "context" + "log" + + "github.com/cinode/go/pkg/cmd/static_datastore" +) func main() { - static_datastore.Execute() + if err := static_datastore.Execute(context.Background()); err != nil { + log.Fatal(err.Error()) + } } diff --git a/pkg/cmd/static_datastore/compile.go b/pkg/cmd/static_datastore/compile.go index 363b231..b0d1bdf 100644 --- a/pkg/cmd/static_datastore/compile.go +++ b/pkg/cmd/static_datastore/compile.go @@ -46,16 +46,15 @@ func compileCmd() *cobra.Command { "a content with static files that can then be used to serve through a", "simple http server.", }, "\n"), - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { if o.srcDir == "" || o.dstLocation == "" { - cmd.Help() - return + return cmd.Help() } - enc := json.NewEncoder(os.Stdout) + enc := json.NewEncoder(cmd.OutOrStdout()) enc.SetIndent("", " ") - fatalResult := func(format string, args ...interface{}) { + fatalResult := func(format string, args ...interface{}) error { msg := fmt.Sprintf(format, args...) enc.Encode(map[string]string{ @@ -63,23 +62,25 @@ func compileCmd() *cobra.Command { "msg": msg, }) - log.Fatalf(msg) + cmd.SilenceUsage = true + cmd.SilenceErrors = true + return errors.New(msg) } if len(rootWriterInfoFile) > 0 { data, err := os.ReadFile(rootWriterInfoFile) if err != nil { - fatalResult("Couldn't read data from the writer info file at '%s': %v", rootWriterInfoFile, err) + return fatalResult("Couldn't read data from the writer info file at '%s': %v", rootWriterInfoFile, err) } if len(data) == 0 { - fatalResult("Writer info file at '%s' is empty", rootWriterInfoFile) + return fatalResult("Writer info file at '%s' is empty", rootWriterInfoFile) } rootWriterInfoStr = string(data) } if len(rootWriterInfoStr) > 0 { wi, err := cinodefs.WriterInfoFromString(rootWriterInfoStr) if err != nil { - fatalResult("Couldn't parse writer info: %v", err) + return fatalResult("Couldn't parse writer info: %v", err) } o.writerInfo = wi } @@ -91,7 +92,7 @@ func compileCmd() *cobra.Command { ep, wi, err := compileFS(cmd.Context(), o) if err != nil { - fatalResult("%s", err) + return fatalResult("%s", err) } result := map[string]string{ @@ -104,6 +105,7 @@ func compileCmd() *cobra.Command { enc.Encode(result) log.Println("DONE") + return nil }, } diff --git a/pkg/cmd/static_datastore/root.go b/pkg/cmd/static_datastore/root.go index ddd435a..38b9514 100644 --- a/pkg/cmd/static_datastore/root.go +++ b/pkg/cmd/static_datastore/root.go @@ -1,5 +1,5 @@ /* -Copyright © 2022 Bartłomiej Święcki (byo) +Copyright © 2023 Bartłomiej Święcki (byo) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,8 +17,7 @@ limitations under the License. package static_datastore import ( - "fmt" - "os" + "context" "github.com/spf13/cobra" ) @@ -50,9 +49,6 @@ node is stored in a plaintext in a file called 'entrypoint.txt'. // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - if err := rootCmd().Execute(); err != nil { - fmt.Println(err) - os.Exit(1) - } +func Execute(ctx context.Context) error { + return rootCmd().ExecuteContext(ctx) } diff --git a/pkg/cmd/static_datastore/static_datastore_test.go b/pkg/cmd/static_datastore/static_datastore_test.go index 00a710c..f1d3aeb 100644 --- a/pkg/cmd/static_datastore/static_datastore_test.go +++ b/pkg/cmd/static_datastore/static_datastore_test.go @@ -19,6 +19,7 @@ package static_datastore import ( "bytes" "context" + "encoding/json" "io" "net/http" "net/http/httptest" @@ -30,6 +31,8 @@ import ( "github.com/cinode/go/pkg/cinodefs" "github.com/cinode/go/pkg/cinodefs/httphandler" "github.com/cinode/go/pkg/datastore" + "github.com/cinode/go/pkg/utilities/golang" + "github.com/spf13/cobra" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "golang.org/x/exp/slog" @@ -47,7 +50,6 @@ type CompileAndReadTestSuite struct { } func TestCompileAndReadTestSuite(t *testing.T) { - s := &CompileAndReadTestSuite{ initialTestDataset: []datasetFile{ { @@ -99,39 +101,60 @@ func TestCompileAndReadTestSuite(t *testing.T) { suite.Run(t, s) } +type testOutputParser struct { + Result string `json:"result"` + Msg string `json:"msg"` + WI string `json:"writer-info"` + EP string `json:"entrypoint"` +} + func (s *CompileAndReadTestSuite) uploadDatasetToDatastore( + t *testing.T, dataset []datasetFile, datastoreDir string, - wi *cinodefs.WriterInfo, -) (*cinodefs.WriterInfo, *cinodefs.Entrypoint) { + extraArgs ...string, +) (wi *cinodefs.WriterInfo, ep *cinodefs.Entrypoint) { + dir := t.TempDir() + + for _, td := range dataset { + err := os.MkdirAll(filepath.Join(dir, filepath.Dir(td.fName)), 0777) + s.Require().NoError(err) - var ep *cinodefs.Entrypoint - s.T().Run("prepare dataset", func(t *testing.T) { + err = os.WriteFile(filepath.Join(dir, td.fName), []byte(td.contents), 0600) + s.Require().NoError(err) + } - dir := t.TempDir() + buf := bytes.NewBuffer(nil) - for _, td := range dataset { - err := os.MkdirAll(filepath.Join(dir, filepath.Dir(td.fName)), 0777) - s.Require().NoError(err) + args := []string{ + "compile", + "-s", dir, + "-d", datastoreDir, + } + args = append(args, extraArgs...) - err = os.WriteFile(filepath.Join(dir, td.fName), []byte(td.contents), 0600) - s.Require().NoError(err) - } + cmd := rootCmd() + cmd.SetArgs(args) + cmd.SetOut(buf) - retEp, retWi, err := compileFS(context.Background(), compileFSOptions{ - srcDir: dir, - dstLocation: datastoreDir, - writerInfo: wi, - }) - require.NoError(t, err) - wi = retWi - ep = retEp - }) + err := cmd.Execute() + require.NoError(t, err) + output := testOutputParser{} + + err = json.Unmarshal(buf.Bytes(), &output) + require.NoError(t, err) + require.Equal(t, "OK", output.Result) + + if output.WI != "" { + wi = golang.Must(cinodefs.WriterInfoFromString(output.WI)) + } + ep = golang.Must(cinodefs.EntrypointFromString(output.EP)) return wi, ep } func (s *CompileAndReadTestSuite) validateDataset( + t *testing.T, dataset []datasetFile, ep *cinodefs.Entrypoint, datastoreDir string, @@ -155,64 +178,181 @@ func (s *CompileAndReadTestSuite) validateDataset( defer testServer.Close() for _, td := range dataset { - s.Run(td.fName, func() { + t.Run(td.fName, func(t *testing.T) { res, err := http.Get(testServer.URL + td.fName) - s.Require().NoError(err) + require.NoError(t, err) defer res.Body.Close() data, err := io.ReadAll(res.Body) - s.Require().NoError(err) - s.Require().Equal([]byte(td.contents), data) + require.NoError(t, err) + require.Equal(t, []byte(td.contents), data) res, err = http.Post(testServer.URL+td.fName, "plain/text", bytes.NewReader([]byte("test"))) - s.Require().NoError(err) + require.NoError(t, err) defer res.Body.Close() - s.Require().Equal(http.StatusMethodNotAllowed, res.StatusCode) + require.Equal(t, http.StatusMethodNotAllowed, res.StatusCode) res, err = http.Get(testServer.URL + td.fName + ".notfound") - s.Require().NoError(err) + require.NoError(t, err) defer res.Body.Close() - s.Require().Equal(http.StatusNotFound, res.StatusCode) + require.Equal(t, http.StatusNotFound, res.StatusCode) }) } - s.Run("Default to index.html", func() { + t.Run("Default to index.html", func(t *testing.T) { res, err := http.Get(testServer.URL + "/") - s.Require().NoError(err) + require.NoError(t, err) defer res.Body.Close() data, err := io.ReadAll(res.Body) - s.Require().NoError(err) + require.NoError(t, err) - s.Require().Equal([]byte("Index"), data) + require.Equal(t, []byte("Index"), data) }) } func (s *CompileAndReadTestSuite) TestCompileAndRead() { - datastore := s.T().TempDir() + t := s.T() + datastore := t.TempDir() // Create and test initial dataset - wi, ep := s.uploadDatasetToDatastore(s.initialTestDataset, datastore, nil) - s.validateDataset(s.initialTestDataset, ep, datastore) + wi, ep := s.uploadDatasetToDatastore(t, s.initialTestDataset, datastore) + s.validateDataset(t, s.initialTestDataset, ep, datastore) + + t.Run("Re-upload same dataset", func(t *testing.T) { + s.uploadDatasetToDatastore(t, s.initialTestDataset, datastore, + "--writer-info", wi.String(), + ) + s.validateDataset(t, s.initialTestDataset, ep, datastore) + }) - // Re-upload same dataset - s.uploadDatasetToDatastore(s.initialTestDataset, datastore, wi) - s.validateDataset(s.initialTestDataset, ep, datastore) + t.Run("Upload modified dataset but for different root link", func(t *testing.T) { + _, updatedEP := s.uploadDatasetToDatastore(t, s.updatedTestDataset, datastore) + s.validateDataset(t, s.updatedTestDataset, updatedEP, datastore) + s.Require().NotEqual(ep, updatedEP) - // Upload modified dataset but for different root link - _, updatedEP := s.uploadDatasetToDatastore(s.updatedTestDataset, datastore, nil) - s.validateDataset(s.updatedTestDataset, updatedEP, datastore) - s.Require().NotEqual(ep, updatedEP) + // After restoring the original entrypoint dataset should be back to the initial one + s.validateDataset(t, s.initialTestDataset, ep, datastore) + }) - // After restoring the original entrypoint dataset should be back to the initial one - s.validateDataset(s.initialTestDataset, ep, datastore) + t.Run("Update the original entrypoint with the new dataset", func(t *testing.T) { + _, epOrigWriterInfo := s.uploadDatasetToDatastore(t, s.updatedTestDataset, datastore, + "--writer-info", wi.String(), + ) + s.validateDataset(t, s.updatedTestDataset, epOrigWriterInfo, datastore) - // Update the original entrypoint with the new dataset - _, epOrigWriterInfo := s.uploadDatasetToDatastore(s.updatedTestDataset, datastore, wi) - s.validateDataset(s.updatedTestDataset, epOrigWriterInfo, datastore) + // Entrypoint must stay the same + require.EqualValues(t, ep, epOrigWriterInfo) + }) - // Entrypoint must stay the same - s.Require().EqualValues(ep, epOrigWriterInfo) + s.T().Run("Upload data with static entrypoint", func(t *testing.T) { + wiStatic, epStatic := s.uploadDatasetToDatastore(t, s.initialTestDataset, datastore, + "--static", + ) + s.validateDataset(t, s.initialTestDataset, epStatic, datastore) + require.Nil(t, wiStatic) + }) + + s.T().Run("Read writer info from file", func(t *testing.T) { + wiFile := filepath.Join(t.TempDir(), "epfile") + require.NoError(t, os.WriteFile(wiFile, []byte(wi.String()), 0777)) + + _, ep := s.uploadDatasetToDatastore(t, s.initialTestDataset, datastore, + "--writer-info-file", wiFile, + ) + s.validateDataset(t, s.initialTestDataset, ep, datastore) + }) + +} + +func testExecCommand(cmd *cobra.Command, args []string) (output, stderr []byte, err error) { + outputBuff := bytes.NewBuffer(nil) + stderrBuff := bytes.NewBuffer(nil) + cmd.SetOutput(outputBuff) + cmd.SetErr(stderrBuff) + cmd.SetArgs(args) + err = cmd.Execute() + return outputBuff.Bytes(), stderrBuff.Bytes(), err +} + +func testExec(args []string) (output, stderr []byte, err error) { + return testExecCommand(rootCmd(), args) +} + +func TestHelpCalls(t *testing.T) { + for _, d := range []struct { + name string + args []string + }{ + {"no args", []string{}}, + {"not enough compile args", []string{"compile"}}, + } { + t.Run(d.name, func(t *testing.T) { + cmd := rootCmd() + helpCalled := false + cmd.SetHelpFunc(func(c *cobra.Command, s []string) { helpCalled = true }) + cmd.SetArgs(d.args) + err := cmd.Execute() + require.NoError(t, err) + require.True(t, helpCalled) + }) + } +} + +func TestInvalidOptions(t *testing.T) { + tempDir := t.TempDir() + emptyFile := filepath.Join(tempDir, "empty") + + err := os.WriteFile(emptyFile, []byte{}, 0777) + require.NoError(t, err) + + for _, d := range []struct { + name string + args []string + errorContains string + }{ + { + name: "invalid root writer info", + args: []string{ + "compile", + "--source", t.TempDir(), + "--destination", t.TempDir(), + "--writer-info", "not-a-valid-writer-info", + }, + errorContains: "Couldn't parse writer info:", + }, + { + name: "invalid root writer info file", + args: []string{ + "compile", + "--source", t.TempDir(), + "--destination", t.TempDir(), + "--writer-info-file", "/invalid/file/name/with/writer/info", + }, + errorContains: "no such file or directory", + }, + { + name: "empty root writer info file", + args: []string{ + "compile", + "--source", t.TempDir(), + "--destination", t.TempDir(), + "--writer-info-file", emptyFile, + }, + errorContains: "is empty", + }, + } { + t.Run(d.name, func(t *testing.T) { + output, _, err := testExec(d.args) + require.ErrorContains(t, err, d.errorContains) + + parsedOutput := testOutputParser{} + err = json.Unmarshal(output, &parsedOutput) + require.NoError(t, err) + require.Equal(t, "ERROR", parsedOutput.Result) + require.Contains(t, parsedOutput.Msg, d.errorContains) + }) + } } From 9344115de7130f33ba1bd7a3667478aa752aaa08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Sun, 19 Nov 2023 19:06:20 +0100 Subject: [PATCH 27/29] Allow configuring listen port --- pkg/cmd/cinode_web_proxy/root.go | 32 ++++++++++++------- pkg/cmd/cinode_web_proxy/root_test.go | 25 ++++++++++++--- pkg/cmd/public_node/root.go | 30 +++++++++++++---- pkg/cmd/public_node/root_test.go | 46 +++++++++++++++++++++------ pkg/utilities/golang/assert_test.go | 16 ++++++++++ 5 files changed, 118 insertions(+), 31 deletions(-) create mode 100644 pkg/utilities/golang/assert_test.go diff --git a/pkg/cmd/cinode_web_proxy/root.go b/pkg/cmd/cinode_web_proxy/root.go index 274d522..be555a4 100644 --- a/pkg/cmd/cinode_web_proxy/root.go +++ b/pkg/cmd/cinode_web_proxy/root.go @@ -25,6 +25,7 @@ import ( "os" "runtime" "sort" + "strconv" "strings" "time" @@ -32,6 +33,7 @@ import ( "github.com/cinode/go/pkg/cinodefs" "github.com/cinode/go/pkg/cinodefs/httphandler" "github.com/cinode/go/pkg/datastore" + "github.com/cinode/go/pkg/utilities/golang" "github.com/cinode/go/pkg/utilities/httpserver" "golang.org/x/exp/slog" ) @@ -79,10 +81,7 @@ func executeWithConfig(ctx context.Context, cfg *config) error { "cpus", runtime.NumCPU(), ) - handler, err := setupCinodeProxy(ctx, mainDS, additionalDSs, entrypoint) - if err != nil { - return err - } + handler := setupCinodeProxy(ctx, mainDS, additionalDSs, entrypoint) return httpserver.RunGracefully(ctx, handler, @@ -96,8 +95,8 @@ func setupCinodeProxy( mainDS datastore.DS, additionalDSs []datastore.DS, entrypoint *cinodefs.Entrypoint, -) (http.Handler, error) { - fs, err := cinodefs.New( +) http.Handler { + fs := golang.Must(cinodefs.New( ctx, blenc.FromDatastore( datastore.NewMultiSource( @@ -108,16 +107,13 @@ func setupCinodeProxy( ), cinodefs.RootEntrypoint(entrypoint), cinodefs.MaxLinkRedirects(10), - ) - if err != nil { - return nil, err - } + )) return &httphandler.Handler{ FS: fs, IndexFile: "index.html", Log: slog.Default(), - }, nil + } } type config struct { @@ -163,7 +159,19 @@ func getConfig() (*config, error) { cfg.additionalDSLocations = append(cfg.additionalDSLocations, location) } - cfg.port = 8080 + port := os.Getenv("CINODE_LISTEN_PORT") + if port == "" { + cfg.port = 8080 + } else { + portNum, err := strconv.Atoi(port) + if err == nil && (portNum < 1 || portNum > 65535) { + err = fmt.Errorf("not in range 1..65535") + } + if err != nil { + return nil, fmt.Errorf("invalid listen port %s: %w", port, err) + } + cfg.port = portNum + } return &cfg, nil } diff --git a/pkg/cmd/cinode_web_proxy/root_test.go b/pkg/cmd/cinode_web_proxy/root_test.go index afb1930..5e2bb3e 100644 --- a/pkg/cmd/cinode_web_proxy/root_test.go +++ b/pkg/cmd/cinode_web_proxy/root_test.go @@ -103,6 +103,25 @@ func TestGetConfig(t *testing.T) { "additional3", }) }) + + t.Run("set listen port", func(t *testing.T) { + t.Setenv("CINODE_LISTEN_PORT", "12345") + cfg, err := getConfig() + require.NoError(t, err) + require.Equal(t, 12345, cfg.port) + }) + + t.Run("invalid port - not a number", func(t *testing.T) { + t.Setenv("CINODE_LISTEN_PORT", "123-45") + _, err := getConfig() + require.ErrorContains(t, err, "invalid listen port") + }) + + t.Run("invalid port - outside range", func(t *testing.T) { + t.Setenv("CINODE_LISTEN_PORT", "-1") + _, err := getConfig() + require.ErrorContains(t, err, "invalid listen port") + }) } func TestWebProxyHandlerInvalidEntrypoint(t *testing.T) { @@ -114,13 +133,12 @@ func TestWebProxyHandlerInvalidEntrypoint(t *testing.T) { key := cipherfactory.NewKeyGenerator(blobtypes.Static).Generate() - handler, err := setupCinodeProxy( + handler := setupCinodeProxy( context.Background(), datastore.InMemory(), []datastore.DS{}, cinodefs.EntrypointFromBlobNameAndKey(n, key), ) - require.NoError(t, err) server := httptest.NewServer(handler) defer server.Close() @@ -179,8 +197,7 @@ func TestWebProxyHandlerSimplePage(t *testing.T) { return ep }() - handler, err := setupCinodeProxy(context.Background(), ds, []datastore.DS{}, ep) - require.NoError(t, err) + handler := setupCinodeProxy(context.Background(), ds, []datastore.DS{}, ep) server := httptest.NewServer(handler) defer server.Close() diff --git a/pkg/cmd/public_node/root.go b/pkg/cmd/public_node/root.go index 9060291..0f19d82 100644 --- a/pkg/cmd/public_node/root.go +++ b/pkg/cmd/public_node/root.go @@ -25,6 +25,7 @@ import ( "os" "runtime" "sort" + "strconv" "strings" "time" @@ -34,10 +35,14 @@ import ( ) func Execute(ctx context.Context) error { - return executeWithConfig(ctx, getConfig()) + cfg, err := getConfig() + if err != nil { + return err + } + return executeWithConfig(ctx, cfg) } -func executeWithConfig(ctx context.Context, cfg config) error { +func executeWithConfig(ctx context.Context, cfg *config) error { handler, err := buildHttpHandler(cfg) if err != nil { return err @@ -60,7 +65,7 @@ func executeWithConfig(ctx context.Context, cfg config) error { ) } -func buildHttpHandler(cfg config) (http.Handler, error) { +func buildHttpHandler(cfg *config) (http.Handler, error) { mainDS, err := datastore.FromLocation(cfg.mainDSLocation) if err != nil { return nil, fmt.Errorf("could not create main datastore: %w", err) @@ -140,7 +145,7 @@ type config struct { uploadPassword string } -func getConfig() config { +func getConfig() (*config, error) { cfg := config{ log: slog.Default(), } @@ -164,9 +169,22 @@ func getConfig() config { cfg.additionalDSLocations = append(cfg.additionalDSLocations, location) } - cfg.port = 8080 + port := os.Getenv("CINODE_LISTEN_PORT") + if port == "" { + cfg.port = 8080 + } else { + portNum, err := strconv.Atoi(port) + if err == nil && (portNum < 1 || portNum > 65535) { + err = fmt.Errorf("not in range 1..65535") + } + if err != nil { + return nil, fmt.Errorf("invalid listen port %s: %w", port, err) + } + cfg.port = portNum + } + cfg.uploadUsername = os.Getenv("CINODE_UPLOAD_USERNAME") cfg.uploadPassword = os.Getenv("CINODE_UPLOAD_PASSWORD") - return cfg + return &cfg, nil } diff --git a/pkg/cmd/public_node/root_test.go b/pkg/cmd/public_node/root_test.go index c9fee86..3e7f4df 100644 --- a/pkg/cmd/public_node/root_test.go +++ b/pkg/cmd/public_node/root_test.go @@ -32,7 +32,8 @@ func TestGetConfig(t *testing.T) { os.Clearenv() t.Run("default config", func(t *testing.T) { - cfg := getConfig() + cfg, err := getConfig() + require.NoError(t, err) require.Equal(t, "memory://", cfg.mainDSLocation) require.Empty(t, cfg.additionalDSLocations) require.Equal(t, 8080, cfg.port) @@ -40,7 +41,8 @@ func TestGetConfig(t *testing.T) { t.Run("set main datastore", func(t *testing.T) { t.Setenv("CINODE_MAIN_DATASTORE", "testdatastore") - cfg := getConfig() + cfg, err := getConfig() + require.NoError(t, err) require.Equal(t, cfg.mainDSLocation, "testdatastore") }) @@ -50,7 +52,8 @@ func TestGetConfig(t *testing.T) { t.Setenv("CINODE_ADDITIONAL_DATASTORE_2", "additional2") t.Setenv("CINODE_ADDITIONAL_DATASTORE_1", "additional1") - cfg := getConfig() + cfg, err := getConfig() + require.NoError(t, err) require.Equal(t, cfg.additionalDSLocations, []string{ "additional", "additional1", @@ -58,11 +61,30 @@ func TestGetConfig(t *testing.T) { "additional3", }) }) + + t.Run("set listen port", func(t *testing.T) { + t.Setenv("CINODE_LISTEN_PORT", "12345") + cfg, err := getConfig() + require.NoError(t, err) + require.Equal(t, 12345, cfg.port) + }) + + t.Run("invalid port - not a number", func(t *testing.T) { + t.Setenv("CINODE_LISTEN_PORT", "123-45") + _, err := getConfig() + require.ErrorContains(t, err, "invalid listen port") + }) + + t.Run("invalid port - outside range", func(t *testing.T) { + t.Setenv("CINODE_LISTEN_PORT", "-1") + _, err := getConfig() + require.ErrorContains(t, err, "invalid listen port") + }) } func TestBuildHttpHandler(t *testing.T) { t.Run("Successfully created handler", func(t *testing.T) { - h, err := buildHttpHandler(config{ + h, err := buildHttpHandler(&config{ mainDSLocation: t.TempDir(), additionalDSLocations: []string{ t.TempDir(), @@ -92,7 +114,7 @@ func TestBuildHttpHandler(t *testing.T) { const VALID_PASSWORD = "secret" const INVALID_PASSWORD = "plaintext" - h, err := buildHttpHandler(config{ + h, err := buildHttpHandler(&config{ mainDSLocation: t.TempDir(), additionalDSLocations: []string{ t.TempDir(), @@ -128,7 +150,7 @@ func TestBuildHttpHandler(t *testing.T) { }) t.Run("invalid main datastore", func(t *testing.T) { - h, err := buildHttpHandler(config{ + h, err := buildHttpHandler(&config{ mainDSLocation: "", }) require.ErrorContains(t, err, "could not create main datastore") @@ -136,7 +158,7 @@ func TestBuildHttpHandler(t *testing.T) { }) t.Run("invalid additional datastore", func(t *testing.T) { - h, err := buildHttpHandler(config{ + h, err := buildHttpHandler(&config{ mainDSLocation: "memory://", additionalDSLocations: []string{""}, }) @@ -152,7 +174,7 @@ func TestExecuteWithConfig(t *testing.T) { time.Sleep(10 * time.Millisecond) cancel() }() - err := executeWithConfig(ctx, config{ + err := executeWithConfig(ctx, &config{ mainDSLocation: "memory://", log: slog.Default(), }) @@ -160,7 +182,7 @@ func TestExecuteWithConfig(t *testing.T) { }) t.Run("invalid configuration", func(t *testing.T) { - err := executeWithConfig(context.Background(), config{}) + err := executeWithConfig(context.Background(), &config{}) require.ErrorContains(t, err, "datastore") }) } @@ -181,4 +203,10 @@ func TestExecute(t *testing.T) { err := Execute(context.Background()) require.ErrorContains(t, err, "datastore") }) + + t.Run("invalid configuration - port", func(t *testing.T) { + t.Setenv("CINODE_LISTEN_PORT", "-1") + err := Execute(context.Background()) + require.ErrorContains(t, err, "listen port") + }) } diff --git a/pkg/utilities/golang/assert_test.go b/pkg/utilities/golang/assert_test.go new file mode 100644 index 0000000..23a4800 --- /dev/null +++ b/pkg/utilities/golang/assert_test.go @@ -0,0 +1,16 @@ +package golang + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAssert(t *testing.T) { + require.NotPanics(t, func() { + Assert(true, "must not happen") + }) + require.Panics(t, func() { + Assert(false, "must panic") + }) +} From 245b332f162cac5cad0a2325d2fd4847c43f6ad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Sun, 19 Nov 2023 19:19:02 +0100 Subject: [PATCH 28/29] Missing copyright header in assert utility function --- pkg/utilities/golang/assert.go | 16 ++++++++++++++++ pkg/utilities/golang/assert_test.go | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/pkg/utilities/golang/assert.go b/pkg/utilities/golang/assert.go index 128600d..220e3c0 100644 --- a/pkg/utilities/golang/assert.go +++ b/pkg/utilities/golang/assert.go @@ -1,3 +1,19 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package golang func Assert(b bool, message string) { diff --git a/pkg/utilities/golang/assert_test.go b/pkg/utilities/golang/assert_test.go index 23a4800..55b3879 100644 --- a/pkg/utilities/golang/assert_test.go +++ b/pkg/utilities/golang/assert_test.go @@ -1,3 +1,19 @@ +/* +Copyright © 2023 Bartłomiej Święcki (byo) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package golang import ( From 4fa3eb505f6b25691c890c670e322b70585f72b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Sun, 19 Nov 2023 19:25:49 +0100 Subject: [PATCH 29/29] Use random port in tests --- pkg/cmd/cinode_web_proxy/root.go | 4 ++-- pkg/cmd/cinode_web_proxy/root_test.go | 1 + pkg/cmd/public_node/root.go | 4 ++-- pkg/cmd/public_node/root_test.go | 2 ++ 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/cmd/cinode_web_proxy/root.go b/pkg/cmd/cinode_web_proxy/root.go index be555a4..c3d4658 100644 --- a/pkg/cmd/cinode_web_proxy/root.go +++ b/pkg/cmd/cinode_web_proxy/root.go @@ -164,8 +164,8 @@ func getConfig() (*config, error) { cfg.port = 8080 } else { portNum, err := strconv.Atoi(port) - if err == nil && (portNum < 1 || portNum > 65535) { - err = fmt.Errorf("not in range 1..65535") + if err == nil && (portNum < 0 || portNum > 65535) { + err = fmt.Errorf("not in range 0..65535") } if err != nil { return nil, fmt.Errorf("invalid listen port %s: %w", port, err) diff --git a/pkg/cmd/cinode_web_proxy/root_test.go b/pkg/cmd/cinode_web_proxy/root_test.go index 5e2bb3e..f55f8ba 100644 --- a/pkg/cmd/cinode_web_proxy/root_test.go +++ b/pkg/cmd/cinode_web_proxy/root_test.go @@ -289,6 +289,7 @@ func TestExecute(t *testing.T) { ep := testblobs.DynamicLink.Entrypoint() t.Setenv("CINODE_ENTRYPOINT", ep.String()) + t.Setenv("CINODE_LISTEN_PORT", "0") ctx, cancel := context.WithCancel(context.Background()) go func() { time.Sleep(10 * time.Millisecond) diff --git a/pkg/cmd/public_node/root.go b/pkg/cmd/public_node/root.go index 0f19d82..e326a73 100644 --- a/pkg/cmd/public_node/root.go +++ b/pkg/cmd/public_node/root.go @@ -174,8 +174,8 @@ func getConfig() (*config, error) { cfg.port = 8080 } else { portNum, err := strconv.Atoi(port) - if err == nil && (portNum < 1 || portNum > 65535) { - err = fmt.Errorf("not in range 1..65535") + if err == nil && (portNum < 0 || portNum > 65535) { + err = fmt.Errorf("not in range 0..65535") } if err != nil { return nil, fmt.Errorf("invalid listen port %s: %w", port, err) diff --git a/pkg/cmd/public_node/root_test.go b/pkg/cmd/public_node/root_test.go index 3e7f4df..105a0b2 100644 --- a/pkg/cmd/public_node/root_test.go +++ b/pkg/cmd/public_node/root_test.go @@ -189,6 +189,8 @@ func TestExecuteWithConfig(t *testing.T) { func TestExecute(t *testing.T) { t.Run("valid configuration", func(t *testing.T) { + t.Setenv("CINODE_LISTEN_PORT", "0") + ctx, cancel := context.WithCancel(context.Background()) go func() { time.Sleep(10 * time.Millisecond)