aliyun-oss-go-sdk/oss/crypto/crypto_bucket.go

package osscrypto

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"hash"
	"hash/crc64"
	"io"
	"net/http"
	"os"
	"strconv"

	kms "github.com/aliyun/alibaba-cloud-sdk-go/services/kms"
	"github.com/aliyun/aliyun-oss-go-sdk/oss"
)

// MasterCipherManager is interface for getting master key with MatDesc(material desc)
// If you may use different master keys for encrypting and decrypting objects,each master
// key must have a unique, non-emtpy, unalterable MatDesc(json string format) and you must provide this interface
// If you always use the same master key for encrypting and decrypting objects, MatDesc
// can be empty and you don't need to provide this interface
//
// matDesc map[string]string:is converted by matDesc json string
// return: []string  the secret key information,such as {"rsa-public-key","rsa-private-key"} or {"non-rsa-key"}
type MasterCipherManager interface {
	GetMasterKey(matDesc map[string]string) ([]string, error)
}

// ExtraCipherBuilder is interface for creating a decrypt ContentCipher with Envelope
// If the objects you need to decrypt are neither encrypted with ContentCipherBuilder
// you provided, nor encrypted with rsa and ali kms master keys, you must provide this interface
//
// ContentCipher  the interface used to decrypt objects
type ExtraCipherBuilder interface {
	GetDecryptCipher(envelope Envelope, cm MasterCipherManager) (ContentCipher, error)
}

// CryptoBucketOption CryptoBucket option such as SetAliKmsClient, SetMasterCipherManager, SetDecryptCipherManager.
type CryptoBucketOption func(*CryptoBucket)

// SetAliKmsClient set field AliKmsClient of CryptoBucket
// If the objects you need to decrypt are encrypted with ali kms master key,but not with ContentCipherBuilder
// you provided, you must provide this interface
func SetAliKmsClient(client *kms.Client) CryptoBucketOption {
	return func(bucket *CryptoBucket) {
		bucket.AliKmsClient = client
	}
}

// SetMasterCipherManager set field MasterCipherManager of CryptoBucket
func SetMasterCipherManager(manager MasterCipherManager) CryptoBucketOption {
	return func(bucket *CryptoBucket) {
		bucket.MasterCipherManager = manager
	}
}

// SetExtraCipherBuilder set field ExtraCipherBuilder of CryptoBucket
func SetExtraCipherBuilder(extraBuilder ExtraCipherBuilder) CryptoBucketOption {
	return func(bucket *CryptoBucket) {
		bucket.ExtraCipherBuilder = extraBuilder
	}
}

// DefaultExtraCipherBuilder is Default implementation of the ExtraCipherBuilder for rsa and kms master keys
type DefaultExtraCipherBuilder struct {
	AliKmsClient *kms.Client
}

// GetDecryptCipher is used to get ContentCipher for decrypt object
func (decb *DefaultExtraCipherBuilder) GetDecryptCipher(envelope Envelope, cm MasterCipherManager) (ContentCipher, error) {
	if cm == nil {
		return nil, fmt.Errorf("DefaultExtraCipherBuilder GetDecryptCipher error,MasterCipherManager is nil")
	}

	if envelope.CEKAlg != AesCtrAlgorithm {
		return nil, fmt.Errorf("DefaultExtraCipherBuilder GetDecryptCipher error,not supported content algorithm %s", envelope.CEKAlg)
	}

	if envelope.WrapAlg != RsaCryptoWrap && envelope.WrapAlg != KmsAliCryptoWrap {
		return nil, fmt.Errorf("DefaultExtraCipherBuilder GetDecryptCipher error,not supported envelope wrap algorithm %s", envelope.WrapAlg)
	}

	matDesc := make(map[string]string)
	if envelope.MatDesc != "" {
		err := json.Unmarshal([]byte(envelope.MatDesc), &matDesc)
		if err != nil {
			return nil, err
		}
	}

	masterKeys, err := cm.GetMasterKey(matDesc)
	if err != nil {
		return nil, err
	}

	var contentCipher ContentCipher
	if envelope.WrapAlg == RsaCryptoWrap {
		// for rsa master key
		if len(masterKeys) != 2 {
			return nil, fmt.Errorf("rsa keys count must be 2,now is %d", len(masterKeys))
		}
		rsaCipher, err := CreateMasterRsa(matDesc, masterKeys[0], masterKeys[1])
		if err != nil {
			return nil, err
		}
		aesCtrBuilder := CreateAesCtrCipher(rsaCipher)
		contentCipher, err = aesCtrBuilder.ContentCipherEnv(envelope)

	} else if envelope.WrapAlg == KmsAliCryptoWrap {
		// for kms master key
		if len(masterKeys) != 1 {
			return nil, fmt.Errorf("non-rsa keys count must be 1,now is %d", len(masterKeys))
		}

		if decb.AliKmsClient == nil {
			return nil, fmt.Errorf("aliyun kms client is nil")
		}

		kmsCipher, err := CreateMasterAliKms(matDesc, masterKeys[0], decb.AliKmsClient)
		if err != nil {
			return nil, err
		}
		aesCtrBuilder := CreateAesCtrCipher(kmsCipher)
		contentCipher, err = aesCtrBuilder.ContentCipherEnv(envelope)
	} else {
		// to do
		// for master keys which are neither rsa nor kms
	}

	return contentCipher, err
}

// CryptoBucket implements the operations for encrypting and decrypting objects
// ContentCipherBuilder is used to encrypt and decrypt objects by default
// when the object's MatDesc which you want to decrypt is emtpy or same to the
// master key's MatDesc you provided in ContentCipherBuilder, sdk try to
// use ContentCipherBuilder to decrypt
type CryptoBucket struct {
	oss.Bucket
	ContentCipherBuilder ContentCipherBuilder
	ExtraCipherBuilder   ExtraCipherBuilder
	MasterCipherManager  MasterCipherManager
	AliKmsClient         *kms.Client
}

// GetCryptoBucket create a client encyrption bucket
func GetCryptoBucket(client *oss.Client, bucketName string, builder ContentCipherBuilder,
	options ...CryptoBucketOption) (*CryptoBucket, error) {
	var cryptoBucket CryptoBucket
	cryptoBucket.Client = *client
	cryptoBucket.BucketName = bucketName
	cryptoBucket.ContentCipherBuilder = builder

	for _, option := range options {
		option(&cryptoBucket)
	}

	if cryptoBucket.ExtraCipherBuilder == nil {
		cryptoBucket.ExtraCipherBuilder = &DefaultExtraCipherBuilder{AliKmsClient: cryptoBucket.AliKmsClient}
	}

	return &cryptoBucket, nil
}

// PutObject creates a new object and encyrpt it on client side when uploading to oss
func (bucket CryptoBucket) PutObject(objectKey string, reader io.Reader, options ...oss.Option) error {
	options = bucket.AddEncryptionUaSuffix(options)
	cc, err := bucket.ContentCipherBuilder.ContentCipher()
	if err != nil {
		return err
	}

	cryptoReader, err := cc.EncryptContent(reader)
	if err != nil {
		return err
	}

	var request *oss.PutObjectRequest
	srcLen, err := oss.GetReaderLen(reader)
	if err != nil {
		request = &oss.PutObjectRequest{
			ObjectKey: objectKey,
			Reader:    cryptoReader,
		}
	} else {
		encryptedLen := cc.GetEncryptedLen(srcLen)
		request = &oss.PutObjectRequest{
			ObjectKey: objectKey,
			Reader:    oss.LimitReadCloser(cryptoReader, encryptedLen),
		}
	}

	opts := addCryptoHeaders(options, cc.GetCipherData())
	resp, err := bucket.DoPutObject(request, opts)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	return err
}

// GetObject downloads the object from oss
// If the object is encrypted, sdk decrypt it automaticly
func (bucket CryptoBucket) GetObject(objectKey string, options ...oss.Option) (io.ReadCloser, error) {
	options = bucket.AddEncryptionUaSuffix(options)
	result, err := bucket.DoGetObject(&oss.GetObjectRequest{ObjectKey: objectKey}, options)
	if err != nil {
		return nil, err
	}
	return result.Response, nil
}

// GetObjectToFile downloads the object from oss to local file
// If the object is encrypted, sdk decrypt it automaticly
func (bucket CryptoBucket) GetObjectToFile(objectKey, filePath string, options ...oss.Option) error {
	options = bucket.AddEncryptionUaSuffix(options)
	tempFilePath := filePath + oss.TempFileSuffix

	// Calls the API to actually download the object. Returns the result instance.
	result, err := bucket.DoGetObject(&oss.GetObjectRequest{ObjectKey: objectKey}, options)
	if err != nil {
		return err
	}
	defer result.Response.Close()

	// If the local file does not exist, create a new one. If it exists, overwrite it.
	fd, err := os.OpenFile(tempFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, oss.FilePermMode)
	if err != nil {
		return err
	}

	// Copy the data to the local file path.
	_, err = io.Copy(fd, result.Response.Body)
	fd.Close()
	if err != nil {
		return err
	}

	// Compares the CRC value
	hasRange, _, _ := oss.IsOptionSet(options, oss.HTTPHeaderRange)
	encodeOpt, _ := oss.FindOption(options, oss.HTTPHeaderAcceptEncoding, nil)
	acceptEncoding := ""
	if encodeOpt != nil {
		acceptEncoding = encodeOpt.(string)
	}
	if bucket.GetConfig().IsEnableCRC && !hasRange && acceptEncoding != "gzip" {
		result.Response.ClientCRC = result.ClientCRC.Sum64()
		err = oss.CheckCRC(result.Response, "GetObjectToFile")
		if err != nil {
			os.Remove(tempFilePath)
			return err
		}
	}

	return os.Rename(tempFilePath, filePath)
}

// DoGetObject is the actual API that gets the encrypted or not encrypted object.
// It's the internal function called by other public APIs.
func (bucket CryptoBucket) DoGetObject(request *oss.GetObjectRequest, options []oss.Option) (*oss.GetObjectResult, error) {
	options = bucket.AddEncryptionUaSuffix(options)

	// first,we must head object
	metaInfo, err := bucket.GetObjectDetailedMeta(request.ObjectKey)
	if err != nil {
		return nil, err
	}

	isEncryptedObj := isEncryptedObject(metaInfo)
	if !isEncryptedObj {
		return bucket.Bucket.DoGetObject(request, options)
	}

	envelope, err := getEnvelopeFromHeader(metaInfo)
	if err != nil {
		return nil, err
	}

	if !isValidContentAlg(envelope.CEKAlg) {
		return nil, fmt.Errorf("not supported content algorithm %s,object:%s", envelope.CEKAlg, request.ObjectKey)
	}

	if !envelope.IsValid() {
		return nil, fmt.Errorf("getEnvelopeFromHeader error,object:%s", request.ObjectKey)
	}

	// use ContentCipherBuilder to decrpt object by default
	encryptMatDesc := bucket.ContentCipherBuilder.GetMatDesc()
	var cc ContentCipher
	err = nil
	if envelope.MatDesc == encryptMatDesc {
		cc, err = bucket.ContentCipherBuilder.ContentCipherEnv(envelope)
	} else {
		cc, err = bucket.ExtraCipherBuilder.GetDecryptCipher(envelope, bucket.MasterCipherManager)
	}

	if err != nil {
		return nil, fmt.Errorf("%s,object:%s", err.Error(), request.ObjectKey)
	}

	discardFrontAlignLen := int64(0)
	uRange, err := oss.GetRangeConfig(options)
	if err != nil {
		return nil, err
	}

	if uRange != nil && uRange.HasStart {
		// process range to align key size
		adjustStart := adjustRangeStart(uRange.Start, cc)
		discardFrontAlignLen = uRange.Start - adjustStart
		if discardFrontAlignLen > 0 {
			uRange.Start = adjustStart
			options = oss.DeleteOption(options, oss.HTTPHeaderRange)
			options = append(options, oss.NormalizedRange(oss.GetRangeString(*uRange)))
		}

		// seek iv
		cipherData := cc.GetCipherData().Clone()
		cipherData.SeekIV(uint64(adjustStart))
		cc, _ = cc.Clone(cipherData)
	}

	params, _ := oss.GetRawParams(options)
	resp, err := bucket.Do("GET", request.ObjectKey, params, options, nil, nil)
	if err != nil {
		return nil, err
	}

	result := &oss.GetObjectResult{
		Response: resp,
	}

	// CRC
	var crcCalc hash.Hash64
	hasRange, _, _ := oss.IsOptionSet(options, oss.HTTPHeaderRange)
	if bucket.GetConfig().IsEnableCRC && !hasRange {
		crcCalc = crc64.New(oss.CrcTable())
		result.ServerCRC = resp.ServerCRC
		result.ClientCRC = crcCalc
	}

	// Progress
	listener := oss.GetProgressListener(options)
	contentLen, _ := strconv.ParseInt(resp.Headers.Get(oss.HTTPHeaderContentLength), 10, 64)
	resp.Body = oss.TeeReader(resp.Body, crcCalc, contentLen, listener, nil)
	resp.Body, err = cc.DecryptContent(resp.Body)
	if err == nil && discardFrontAlignLen > 0 {
		resp.Body = &oss.DiscardReadCloser{
			RC:      resp.Body,
			Discard: int(discardFrontAlignLen)}
	}
	return result, err
}

// PutObjectFromFile creates a new object from the local file
// the object will be encrypted automaticly on client side when uploaded to oss
func (bucket CryptoBucket) PutObjectFromFile(objectKey, filePath string, options ...oss.Option) error {
	options = bucket.AddEncryptionUaSuffix(options)
	fd, err := os.Open(filePath)
	if err != nil {
		return err
	}
	defer fd.Close()

	opts := oss.AddContentType(options, filePath, objectKey)
	cc, err := bucket.ContentCipherBuilder.ContentCipher()
	if err != nil {
		return err
	}

	cryptoReader, err := cc.EncryptContent(fd)
	if err != nil {
		return err
	}

	var request *oss.PutObjectRequest
	srcLen, err := oss.GetReaderLen(fd)
	if err != nil {
		request = &oss.PutObjectRequest{
			ObjectKey: objectKey,
			Reader:    cryptoReader,
		}
	} else {
		encryptedLen := cc.GetEncryptedLen(srcLen)
		request = &oss.PutObjectRequest{
			ObjectKey: objectKey,
			Reader:    oss.LimitReadCloser(cryptoReader, encryptedLen),
		}
	}

	opts = addCryptoHeaders(opts, cc.GetCipherData())
	resp, err := bucket.DoPutObject(request, opts)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	return nil
}

// AppendObject please refer to Bucket.AppendObject
func (bucket CryptoBucket) AppendObject(objectKey string, reader io.Reader, appendPosition int64, options ...oss.Option) (int64, error) {
	return 0, fmt.Errorf("CryptoBucket doesn't support AppendObject")
}

// DoAppendObject please refer to Bucket.DoAppendObject
func (bucket CryptoBucket) DoAppendObject(request *oss.AppendObjectRequest, options []oss.Option) (*oss.AppendObjectResult, error) {
	return nil, fmt.Errorf("CryptoBucket doesn't support DoAppendObject")
}

// PutObjectWithURL please refer to Bucket.PutObjectWithURL
func (bucket CryptoBucket) PutObjectWithURL(signedURL string, reader io.Reader, options ...oss.Option) error {
	return fmt.Errorf("CryptoBucket doesn't support PutObjectWithURL")
}

// PutObjectFromFileWithURL please refer to Bucket.PutObjectFromFileWithURL
func (bucket CryptoBucket) PutObjectFromFileWithURL(signedURL, filePath string, options ...oss.Option) error {
	return fmt.Errorf("CryptoBucket doesn't support PutObjectFromFileWithURL")
}

// DoPutObjectWithURL please refer to Bucket.DoPutObjectWithURL
func (bucket CryptoBucket) DoPutObjectWithURL(signedURL string, reader io.Reader, options []oss.Option) (*oss.Response, error) {
	return nil, fmt.Errorf("CryptoBucket doesn't support DoPutObjectWithURL")
}

// GetObjectWithURL please refer to Bucket.GetObjectWithURL
func (bucket CryptoBucket) GetObjectWithURL(signedURL string, options ...oss.Option) (io.ReadCloser, error) {
	return nil, fmt.Errorf("CryptoBucket doesn't support GetObjectWithURL")
}

// GetObjectToFileWithURL please refer to Bucket.GetObjectToFileWithURL
func (bucket CryptoBucket) GetObjectToFileWithURL(signedURL, filePath string, options ...oss.Option) error {
	return fmt.Errorf("CryptoBucket doesn't support GetObjectToFileWithURL")
}

// DoGetObjectWithURL please refer to Bucket.DoGetObjectWithURL
func (bucket CryptoBucket) DoGetObjectWithURL(signedURL string, options []oss.Option) (*oss.GetObjectResult, error) {
	return nil, fmt.Errorf("CryptoBucket doesn't support DoGetObjectWithURL")
}

// ProcessObject please refer to Bucket.ProcessObject
func (bucket CryptoBucket) ProcessObject(objectKey string, process string, options ...oss.Option) (oss.ProcessObjectResult, error) {
	var out oss.ProcessObjectResult
	return out, fmt.Errorf("CryptoBucket doesn't support ProcessObject")
}

func (bucket CryptoBucket) AddEncryptionUaSuffix(options []oss.Option) []oss.Option {
	var outOption []oss.Option
	bSet, _, _ := oss.IsOptionSet(options, oss.HTTPHeaderUserAgent)
	if bSet || bucket.Client.Config.UserSetUa {
		outOption = options
		return outOption
	}
	outOption = append(options, oss.UserAgentHeader(bucket.Client.Config.UserAgent+"/"+EncryptionUaSuffix))
	return outOption
}

// isEncryptedObject judge the object is encrypted or not
func isEncryptedObject(headers http.Header) bool {
	encrptedKey := headers.Get(oss.HTTPHeaderOssMetaPrefix + OssClientSideEncryptionKey)
	return len(encrptedKey) > 0
}

// addCryptoHeaders save Envelope information in oss meta
func addCryptoHeaders(options []oss.Option, cd *CipherData) []oss.Option {
	opts := []oss.Option{}

	// convert content-md5
	md5Option, _ := oss.FindOption(options, oss.HTTPHeaderContentMD5, nil)
	if md5Option != nil {
		opts = append(opts, oss.Meta(OssClientSideEncryptionUnencryptedContentMD5, md5Option.(string)))
		options = oss.DeleteOption(options, oss.HTTPHeaderContentMD5)
	}

	// convert content-length
	lenOption, _ := oss.FindOption(options, oss.HTTPHeaderContentLength, nil)
	if lenOption != nil {
		opts = append(opts, oss.Meta(OssClientSideEncryptionUnencryptedContentLength, lenOption.(string)))
		options = oss.DeleteOption(options, oss.HTTPHeaderContentLength)
	}

	opts = append(opts, options...)

	// matDesc
	if cd.MatDesc != "" {
		opts = append(opts, oss.Meta(OssClientSideEncryptionMatDesc, cd.MatDesc))
	}

	// encrypted key
	strEncryptedKey := base64.StdEncoding.EncodeToString(cd.EncryptedKey)
	opts = append(opts, oss.Meta(OssClientSideEncryptionKey, strEncryptedKey))

	// encrypted iv
	strEncryptedIV := base64.StdEncoding.EncodeToString(cd.EncryptedIV)
	opts = append(opts, oss.Meta(OssClientSideEncryptionStart, strEncryptedIV))

	// wrap alg
	opts = append(opts, oss.Meta(OssClientSideEncryptionWrapAlg, cd.WrapAlgorithm))

	// cek alg
	opts = append(opts, oss.Meta(OssClientSideEncryptionCekAlg, cd.CEKAlgorithm))

	return opts
}

func getEnvelopeFromHeader(header http.Header) (Envelope, error) {
	var envelope Envelope
	envelope.IV = header.Get(oss.HTTPHeaderOssMetaPrefix + OssClientSideEncryptionStart)
	decodedIV, err := base64.StdEncoding.DecodeString(envelope.IV)
	if err != nil {
		return envelope, err
	}
	envelope.IV = string(decodedIV)

	envelope.CipherKey = header.Get(oss.HTTPHeaderOssMetaPrefix + OssClientSideEncryptionKey)
	decodedKey, err := base64.StdEncoding.DecodeString(envelope.CipherKey)
	if err != nil {
		return envelope, err
	}
	envelope.CipherKey = string(decodedKey)

	envelope.MatDesc = header.Get(oss.HTTPHeaderOssMetaPrefix + OssClientSideEncryptionMatDesc)
	envelope.WrapAlg = header.Get(oss.HTTPHeaderOssMetaPrefix + OssClientSideEncryptionWrapAlg)
	envelope.CEKAlg = header.Get(oss.HTTPHeaderOssMetaPrefix + OssClientSideEncryptionCekAlg)
	envelope.UnencryptedMD5 = header.Get(oss.HTTPHeaderOssMetaPrefix + OssClientSideEncryptionUnencryptedContentMD5)
	envelope.UnencryptedContentLen = header.Get(oss.HTTPHeaderOssMetaPrefix + OssClientSideEncryptionUnencryptedContentLength)
	return envelope, err
}

func isValidContentAlg(algName string) bool {
	// now content encyrption only support aec/ctr algorithm
	return algName == AesCtrAlgorithm
}

func adjustRangeStart(start int64, cc ContentCipher) int64 {
	alignLen := int64(cc.GetAlignLen())
	return (start / alignLen) * alignLen
}