refactor(repository): extract parts repo/content into packages (kopia…

…#2651) - repolog package - blobcrypto package - indexblob package Minor cleanups: - removed dead code - introduced New*() methods for object construction
Rodri9o · Dec 17, 2022 · f8be8f6 · f8be8f6
1 parent 4c0fe39
commit f8be8f6
Show file tree

Hide file tree

Showing 33 changed files with 818 additions and 700 deletions.
diff --git a/cli/command_blob_list.go b/cli/command_blob_list.go
@@ -5,9 +5,10 @@ import (
 	"strings"
 
 	"github.com/kopia/kopia/internal/epoch"
+	"github.com/kopia/kopia/internal/repolog"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/blob"
-	"github.com/kopia/kopia/repo/content"
+	"github.com/kopia/kopia/repo/content/indexblob"
 )
 
 type commandBlobList struct {
@@ -57,15 +58,15 @@ func (c *commandBlobList) run(ctx context.Context, rep repo.DirectRepository) er
 
 func (c *commandBlobList) shouldInclude(b blob.Metadata) bool {
 	if c.dataOnly {
-		if strings.HasPrefix(string(b.BlobID), content.LegacyIndexBlobPrefix) {
+		if strings.HasPrefix(string(b.BlobID), indexblob.V0IndexBlobPrefix) {
 			return false
 		}
 
 		if strings.HasPrefix(string(b.BlobID), epoch.EpochManagerIndexUberPrefix) {
 			return false
 		}
 
-		if strings.HasPrefix(string(b.BlobID), content.TextLogBlobPrefix) {
+		if strings.HasPrefix(string(b.BlobID), repolog.BlobPrefix) {
 			return false
 		}
 

diff --git a/cli/command_blob_show.go b/cli/command_blob_show.go
@@ -8,11 +8,11 @@ import (
 
 	"github.com/pkg/errors"
 
+	"github.com/kopia/kopia/internal/blobcrypto"
 	"github.com/kopia/kopia/internal/gather"
 	"github.com/kopia/kopia/internal/iocopy"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/blob"
-	"github.com/kopia/kopia/repo/content"
 )
 
 type commandBlobShow struct {
@@ -57,7 +57,7 @@ func (c *commandBlobShow) maybeDecryptBlob(ctx context.Context, w io.Writer, rep
 		var tmp gather.WriteBuffer
 		defer tmp.Close()
 
-		if err := content.DecryptBLOB(rep.ContentReader().ContentFormat(), b, blobID, &tmp); err != nil {
+		if err := blobcrypto.Decrypt(rep.ContentReader().ContentFormat(), b, blobID, &tmp); err != nil {
 			return errors.Wrap(err, "error decrypting blob")
 		}
 

diff --git a/cli/command_index_inspect.go b/cli/command_index_inspect.go
@@ -11,6 +11,7 @@ import (
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/blob"
 	"github.com/kopia/kopia/repo/content"
+	"github.com/kopia/kopia/repo/content/indexblob"
 )
 
 type commandIndexInspect struct {
@@ -81,7 +82,7 @@ func (c *commandIndexInspect) inspectAllBlobs(ctx context.Context, rep repo.Dire
 		return errors.Wrap(err, "error listing index blobs")
 	}
 
-	indexesCh := make(chan content.IndexBlobInfo, len(indexes))
+	indexesCh := make(chan indexblob.Metadata, len(indexes))
 	for _, bm := range indexes {
 		indexesCh <- bm
 	}

diff --git a/cli/command_index_optimize.go b/cli/command_index_optimize.go
@@ -5,7 +5,7 @@ import (
 	"time"
 
 	"github.com/kopia/kopia/repo"
-	"github.com/kopia/kopia/repo/content"
+	"github.com/kopia/kopia/repo/content/indexblob"
 )
 
 type commandIndexOptimize struct {
@@ -36,7 +36,7 @@ func (c *commandIndexOptimize) runOptimizeCommand(ctx context.Context, rep repo.
 		return err
 	}
 
-	opt := content.CompactOptions{
+	opt := indexblob.CompactOptions{
 		MaxSmallBlobs: c.optimizeMaxSmallBlobs,
 		AllIndexes:    c.optimizeAllIndexes,
 		DropContents:  contentIDs,

diff --git a/cli/command_index_recover.go b/cli/command_index_recover.go
@@ -12,6 +12,7 @@ import (
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/blob"
 	"github.com/kopia/kopia/repo/content"
+	"github.com/kopia/kopia/repo/content/indexblob"
 )
 
 type commandIndexRecover struct {
@@ -60,7 +61,7 @@ func (c *commandIndexRecover) run(ctx context.Context, rep repo.DirectRepository
 	}()
 
 	if c.deleteIndexes {
-		if err := rep.BlobReader().ListBlobs(ctx, content.LegacyIndexBlobPrefix, func(bm blob.Metadata) error {
+		if err := rep.BlobReader().ListBlobs(ctx, indexblob.V0IndexBlobPrefix, func(bm blob.Metadata) error {
 			if c.commit {
 				log(ctx).Infof("deleting old index blob: %v", bm.BlobID)
 				return errors.Wrap(rep.BlobStorage().DeleteBlob(ctx, bm.BlobID), "error deleting index blob")

diff --git a/cli/command_logs_session.go b/cli/command_logs_session.go
@@ -11,8 +11,8 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/kopia/kopia/internal/clock"
+	"github.com/kopia/kopia/internal/repolog"
 	"github.com/kopia/kopia/repo/blob"
-	"github.com/kopia/kopia/repo/content"
 )
 
 type logSessionInfo struct {
@@ -70,7 +70,7 @@ func getLogSessions(ctx context.Context, st blob.Reader) ([]*logSessionInfo, err
 
 	var allSessions []*logSessionInfo
 
-	if err := st.ListBlobs(ctx, content.TextLogBlobPrefix, func(bm blob.Metadata) error {
+	if err := st.ListBlobs(ctx, repolog.BlobPrefix, func(bm blob.Metadata) error {
 		parts := strings.Split(string(bm.BlobID), "_")
 
 		//nolint:gomnd

diff --git a/cli/command_logs_show.go b/cli/command_logs_show.go
@@ -5,9 +5,9 @@ import (
 
 	"github.com/pkg/errors"
 
+	"github.com/kopia/kopia/internal/blobcrypto"
 	"github.com/kopia/kopia/internal/gather"
 	"github.com/kopia/kopia/repo"
-	"github.com/kopia/kopia/repo/content"
 )
 
 type commandLogsShow struct {
@@ -70,7 +70,7 @@ func (c *commandLogsShow) run(ctx context.Context, rep repo.DirectRepository) er
 				return errors.Wrap(err, "error getting log")
 			}
 
-			if err := content.DecryptBLOB(rep.ContentReader().ContentFormat(), data.Bytes(), bm.BlobID, &decrypted); err != nil {
+			if err := blobcrypto.Decrypt(rep.ContentReader().ContentFormat(), data.Bytes(), bm.BlobID, &decrypted); err != nil {
 				return errors.Wrap(err, "error decrypting log")
 			}
 

diff --git a/cli/command_repository_upgrade.go b/cli/command_repository_upgrade.go
@@ -14,6 +14,7 @@ import (
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/content"
 	"github.com/kopia/kopia/repo/content/index"
+	"github.com/kopia/kopia/repo/content/indexblob"
 	"github.com/kopia/kopia/repo/format"
 )
 
@@ -98,7 +99,7 @@ func assign(iif content.Info, i int, m map[content.ID][2]index.Info) {
 }
 
 // loadIndexBlobs load index blobs into indexEntries map.  indexEntries map will allow comparison betweel two indexes (index at which == 0 and index at which == 1).
-func loadIndexBlobs(ctx context.Context, indexEntries map[content.ID][2]index.Info, sm *content.SharedManager, which int, indexBlobInfos []content.IndexBlobInfo) error {
+func loadIndexBlobs(ctx context.Context, indexEntries map[content.ID][2]index.Info, sm *content.SharedManager, which int, indexBlobInfos []indexblob.Metadata) error {
 	d := gather.WriteBuffer{}
 
 	for _, indexBlobInfo := range indexBlobInfos {

diff --git a/repo/content/blob_crypto.go → internal/blobcrypto/blob_crypto.go b/repo/content/blob_crypto.go → internal/blobcrypto/blob_crypto.go
@@ -1,4 +1,5 @@
-package content
+// Package blobcrypto performs whole-blob crypto operations.
+package blobcrypto
 
 import (
 	"crypto/aes"
@@ -13,8 +14,8 @@ import (
 	"github.com/kopia/kopia/repo/hashing"
 )
 
-// crypter ecapsulates hashing and encryption.
-type crypter interface {
+// Crypter ecapsulates hashing and encryption.
+type Crypter interface {
 	HashFunc() hashing.HashFunc
 	Encryptor() encryption.Encryptor
 }
@@ -38,16 +39,16 @@ func getIndexBlobIV(s blob.ID) ([]byte, error) {
 	return v, nil
 }
 
-// EncryptBLOB encrypts the given data using crypter-defined key and returns a name that should
+// Encrypt encrypts the given data using crypter-defined key and returns a name that should
 // be used to save the blob in thre repository.
-func EncryptBLOB(c crypter, payload gather.Bytes, prefix blob.ID, sessionID SessionID, output *gather.WriteBuffer) (blob.ID, error) {
+func Encrypt(c Crypter, payload gather.Bytes, prefix, suffix blob.ID, output *gather.WriteBuffer) (blob.ID, error) {
 	var hashOutput [hashing.MaxHashSize]byte
 
 	hash := c.HashFunc()(hashOutput[:0], payload)
 	blobID := prefix + blob.ID(hex.EncodeToString(hash))
 
-	if sessionID != "" {
-		blobID += blob.ID("-" + sessionID)
+	if suffix != "" {
+		blobID += "-" + suffix
 	}
 
 	iv, err := getIndexBlobIV(blobID)
@@ -64,8 +65,8 @@ func EncryptBLOB(c crypter, payload gather.Bytes, prefix blob.ID, sessionID Sess
 	return blobID, nil
 }
 
-// DecryptBLOB decrypts the provided data using provided blobID to derive initialization vector.
-func DecryptBLOB(c crypter, payload gather.Bytes, blobID blob.ID, output *gather.WriteBuffer) error {
+// Decrypt decrypts the provided data using provided blobID to derive initialization vector.
+func Decrypt(c Crypter, payload gather.Bytes, blobID blob.ID, output *gather.WriteBuffer) error {
 	iv, err := getIndexBlobIV(blobID)
 	if err != nil {
 		return errors.Wrap(err, "unable to get index blob IV")

diff --git a/repo/content/blob_crypto_test.go → internal/blobcrypto/blob_crypto_test.go b/repo/content/blob_crypto_test.go → internal/blobcrypto/blob_crypto_test.go
@@ -1,4 +1,4 @@
-package content
+package blobcrypto
 
 import (
 	"strings"
@@ -30,31 +30,31 @@ func TestBlobCrypto(t *testing.T) {
 	defer tmp2.Close()
 	defer tmp3.Close()
 
-	id, err := EncryptBLOB(cr, gather.FromSlice([]byte{1, 2, 3}), "n", "mysessionid", &tmp)
+	id, err := Encrypt(cr, gather.FromSlice([]byte{1, 2, 3}), "n", "mysessionid", &tmp)
 	require.NoError(t, err)
 
-	id2, err := EncryptBLOB(cr, gather.FromSlice([]byte{1, 2, 4}), "n", "mysessionid", &tmp2)
+	id2, err := Encrypt(cr, gather.FromSlice([]byte{1, 2, 4}), "n", "mysessionid", &tmp2)
 	require.NoError(t, err)
 
 	require.NotEqual(t, id, id2)
 
-	require.NoError(t, DecryptBLOB(cr, tmp.Bytes(), id, &tmp3))
+	require.NoError(t, Decrypt(cr, tmp.Bytes(), id, &tmp3))
 	require.Equal(t, []byte{1, 2, 3}, tmp3.ToByteSlice())
-	require.NoError(t, DecryptBLOB(cr, tmp2.Bytes(), id2, &tmp3))
+	require.NoError(t, Decrypt(cr, tmp2.Bytes(), id2, &tmp3))
 	require.Equal(t, []byte{1, 2, 4}, tmp3.ToByteSlice())
 
 	// decrypting using invalid ID fails
-	require.Error(t, DecryptBLOB(cr, tmp.Bytes(), id2, &tmp3))
-	require.Error(t, DecryptBLOB(cr, tmp2.Bytes(), id, &tmp3))
+	require.Error(t, Decrypt(cr, tmp.Bytes(), id2, &tmp3))
+	require.Error(t, Decrypt(cr, tmp2.Bytes(), id, &tmp3))
 
 	require.True(t, strings.HasPrefix(string(id), "n"))
 	require.True(t, strings.HasSuffix(string(id), "-mysessionid"), id)
 
 	// negative cases
-	require.Error(t, DecryptBLOB(cr, tmp.Bytes(), "invalid-blob-id", &tmp3))
-	require.Error(t, DecryptBLOB(cr, tmp.Bytes(), "zzz0123456789abcdef0123456789abcde-suffix", &tmp3))
-	require.Error(t, DecryptBLOB(cr, tmp.Bytes(), id2, &tmp3))
-	require.Error(t, DecryptBLOB(cr, gather.FromSlice([]byte{2, 3, 4}), id, &tmp2))
+	require.Error(t, Decrypt(cr, tmp.Bytes(), "invalid-blob-id", &tmp3))
+	require.Error(t, Decrypt(cr, tmp.Bytes(), "zzz0123456789abcdef0123456789abcde-suffix", &tmp3))
+	require.Error(t, Decrypt(cr, tmp.Bytes(), id2, &tmp3))
+	require.Error(t, Decrypt(cr, gather.FromSlice([]byte{2, 3, 4}), id, &tmp2))
 }
 
 type badEncryptor struct{}
@@ -83,7 +83,7 @@ func TestBlobCrypto_Invalid(t *testing.T) {
 	defer tmp2.Close()
 	defer tmp3.Close()
 
-	_, err := EncryptBLOB(cr, gather.FromSlice([]byte{1, 2, 3}), "n", "mysessionid", &tmp)
+	_, err := Encrypt(cr, gather.FromSlice([]byte{1, 2, 3}), "n", "mysessionid", &tmp)
 	require.Error(t, err)
 
 	f := &format.ContentFormat{
@@ -97,6 +97,19 @@ func TestBlobCrypto_Invalid(t *testing.T) {
 
 	cr.h = hf
 
-	_, err = EncryptBLOB(cr, gather.FromSlice([]byte{1, 2, 3}), "n", "mysessionid", &tmp)
+	_, err = Encrypt(cr, gather.FromSlice([]byte{1, 2, 3}), "n", "mysessionid", &tmp)
 	require.Error(t, err)
 }
+
+type staticCrypter struct {
+	h hashing.HashFunc
+	e encryption.Encryptor
+}
+
+func (p staticCrypter) Encryptor() encryption.Encryptor {
+	return p.e
+}
+
+func (p staticCrypter) HashFunc() hashing.HashFunc {
+	return p.h
+}