Merge pull request #144 from oracle/release_2018-11-29

Release 2018 11 29 3.2.0

CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 
+## 3.2.0 - 2018-11-29
+### Added 
+- Support for getting bucket statistics in the Object Storage service
+
+### Fixed
+- Block Storage service for copying volume backups across regions is now enabled 
+- Object storage `PutObject` and `UploadPart` operations now do not override the client's signer
+
 ## 3.1.0 - 2018-11-15
 ### Added
 - Support for VCN transit routing in the Networking service 
@@ -23,7 +31,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
 - Support for generating and downloading wallets in the Database service
 - Support for creating a standalone backup from an on-premises database in the Database service
 - Support for db version and additional connection strings in the Autonomous Transaction Processing and Autonomous Data Warehouse resources of the Database service
-- Support for copying volume backups across regions in the Block Storage service (please see Known issue)
+- Support for copying volume backups across regions in the Block Storage service
 - Support for deleting compartments in the Identity service
 - Support for reboot migration for virtual machines in the Compute service
 - Support for Instance Pools and Instance Configurations in the Compute service
@@ -32,9 +40,6 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
 - The signing algorithm does not lower case the header fields [Github issue 132](https://github.com/oracle/oci-go-sdk/issues/132)
 - Raw configuration provider does not check for empty strings [Github issue 134](https://github.com/oracle/oci-go-sdk/issues/134)
 
-### Known issue
--  Block Storage service for copying volume backups across regions is not enabled
-
 ### Breaking change
 - DbDataSizeInMBs field in Backup and BackupSummary struct was renamed to DatabaseSizeInGBs and type changed from *int to *float64 
     - Before

README.md

@@ -27,6 +27,8 @@ To start working with the Go SDK, you import the service package, create a clien
 ### Configuring 
 Before using the SDK, set up a config file with the required credentials. See [SDK and Tool Configuration](https://docs.us-phoenix-1.oraclecloud.com/Content/API/Concepts/sdkconfig.htm) for instructions.
 
+Note that the Go SDK does not support profile inheritance or defining custom values in the configuration file.
+
 Once a config file has been setup, call `common.DefaultConfigProvider()` function as follows:
 
  ```go

audit/audit_client.go

@@ -37,7 +37,7 @@ func NewAuditClientWithConfigurationProvider(configProvider common.Configuration
 
 // SetRegion overrides the region of this client.
 func (client *AuditClient) SetRegion(region string) {
-	client.Host = fmt.Sprintf(common.DefaultHostURLTemplate, "audit", region)
+	client.Host = common.StringToRegion(region).Endpoint("audit")
 }
 
 // SetConfigurationProvider sets the configuration provider including the region, returns an error if is not valid

common/auth/certificate_retriever.go

@@ -149,10 +149,110 @@ func (r *urlBasedX509CertificateRetriever) PrivateKey() *rsa.PrivateKey {
 	r.mux.Lock()
 	defer r.mux.Unlock()
 
+	//Nil Private keys are supported as part of a certificate
 	if r.privateKey == nil {
 		return nil
 	}
 
 	c := *r.privateKey
 	return &c
 }
+
+//staticCertificateRetriever serves certificates from static data
+type staticCertificateRetriever struct {
+	Passphrase     []byte
+	CertificatePem []byte
+	PrivateKeyPem  []byte
+	certificate    *x509.Certificate
+	privateKey     *rsa.PrivateKey
+	mux            sync.Mutex
+}
+
+//Refresh proccess the inputs into appropiate keys and certificates
+func (r *staticCertificateRetriever) Refresh() error {
+	r.mux.Lock()
+	defer r.mux.Unlock()
+
+	certifcate, err := r.readCertificate()
+	if err != nil {
+		r.certificate = nil
+		return err
+	}
+	r.certificate = certifcate
+
+	key, err := r.readPrivateKey()
+	if err != nil {
+		r.privateKey = nil
+		return err
+	}
+	r.privateKey = key
+
+	return nil
+}
+
+func (r *staticCertificateRetriever) Certificate() *x509.Certificate {
+	r.mux.Lock()
+	defer r.mux.Unlock()
+
+	return r.certificate
+}
+
+func (r *staticCertificateRetriever) PrivateKey() *rsa.PrivateKey {
+	r.mux.Lock()
+	defer r.mux.Unlock()
+
+	return r.privateKey
+}
+
+func (r *staticCertificateRetriever) CertificatePemRaw() []byte {
+	r.mux.Lock()
+	defer r.mux.Unlock()
+
+	if r.CertificatePem == nil {
+		return nil
+	}
+
+	c := make([]byte, len(r.CertificatePem))
+	copy(c, r.CertificatePem)
+	return c
+}
+
+func (r *staticCertificateRetriever) PrivateKeyPemRaw() []byte {
+	r.mux.Lock()
+	defer r.mux.Unlock()
+
+	if r.PrivateKeyPem == nil {
+		return nil
+	}
+
+	c := make([]byte, len(r.PrivateKeyPem))
+	copy(c, r.PrivateKeyPem)
+	return c
+}
+
+func (r *staticCertificateRetriever) readCertificate() (certificate *x509.Certificate, err error) {
+	block, _ := pem.Decode(r.CertificatePem)
+	if block == nil {
+		return nil, fmt.Errorf("failed to parse the new certificate, not valid pem data")
+	}
+
+	if certificate, err = x509.ParseCertificate(block.Bytes); err != nil {
+		return nil, fmt.Errorf("failed to parse the new certificate: %s", err.Error())
+	}
+	return certificate, nil
+}
+
+func (r *staticCertificateRetriever) readPrivateKey() (*rsa.PrivateKey, error) {
+	if r.PrivateKeyPem == nil {
+		return nil, nil
+	}
+
+	var pass *string
+	if r.Passphrase == nil {
+		pass = nil
+	} else {
+		ss := string(r.Passphrase)
+		pass = &ss
+	}
+	return common.PrivateKeyFromBytes(r.PrivateKeyPem, pass)
+}

common/auth/certificate_retriever_test.go

@@ -221,3 +221,35 @@ func generateRandomCertificate() (privateKeyPem, certPem []byte) {
 	certPem = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: newCertBytes})
 	return
 }
+
+func TestStaticCertificateRetriever(t *testing.T) {
+	retriever := staticCertificateRetriever{
+		Passphrase:     []byte(""),
+		CertificatePem: []byte(leafCertPem),
+		PrivateKeyPem:  []byte(leafCertPrivateKeyPem),
+	}
+
+	err := retriever.Refresh()
+	assert.NoError(t, err)
+	key := retriever.PrivateKey()
+	assert.NotNil(t, key)
+	cert := retriever.Certificate()
+	assert.NotNil(t, cert)
+}
+
+func TestBadStaticCertificateRetriever(t *testing.T) {
+	retriever := staticCertificateRetriever{
+		Passphrase:     []byte(""),
+		CertificatePem: []byte(""),
+		PrivateKeyPem:  []byte(""),
+	}
+
+	err := retriever.Refresh()
+	assert.Error(t, err)
+
+	c := retriever.Certificate()
+	assert.Nil(t, c)
+
+	k := retriever.PrivateKey()
+	assert.Nil(t, k)
+}

common/auth/configuration.go

@@ -9,7 +9,7 @@ import (
 )
 
 type instancePrincipalConfigurationProvider struct {
-	keyProvider *instancePrincipalKeyProvider
+	keyProvider instancePrincipalKeyProvider
 	region      *common.Region
 }
 
@@ -20,7 +20,7 @@ func InstancePrincipalConfigurationProvider() (common.ConfigurationProvider, err
 	if keyProvider, err = newInstancePrincipalKeyProvider(); err != nil {
 		return nil, fmt.Errorf("failed to create a new key provider for instance principal: %s", err.Error())
 	}
-	return instancePrincipalConfigurationProvider{keyProvider: keyProvider, region: nil}, nil
+	return instancePrincipalConfigurationProvider{keyProvider: *keyProvider, region: nil}, nil
 }
 
 //InstancePrincipalConfigurationProviderForRegion returns a configuration for instance principals with a given region
@@ -30,7 +30,37 @@ func InstancePrincipalConfigurationProviderForRegion(region common.Region) (comm
 	if keyProvider, err = newInstancePrincipalKeyProvider(); err != nil {
 		return nil, fmt.Errorf("failed to create a new key provider for instance principal: %s", err.Error())
 	}
-	return instancePrincipalConfigurationProvider{keyProvider: keyProvider, region: &region}, nil
+	return instancePrincipalConfigurationProvider{keyProvider: *keyProvider, region: &region}, nil
+}
+
+//InstancePrincipalConfigurationWithCerts returns a configuration for instance principals with a given region and hardcoded certificates in lieu of metadata service certs
+func InstancePrincipalConfigurationWithCerts(region common.Region, leafCertificate, leafPassphrase, leafPrivateKey []byte, intermediateCertificates [][]byte) (common.ConfigurationProvider, error) {
+	leafCertificateRetriever := staticCertificateRetriever{Passphrase: leafPassphrase, CertificatePem: leafCertificate, PrivateKeyPem: leafPrivateKey}
+
+	//The .Refresh() call actually reads the certificates from the inputs
+	err := leafCertificateRetriever.Refresh()
+	if err != nil {
+		return nil, err
+	}
+
+	certificate := leafCertificateRetriever.Certificate()
+
+	tenancyID := extractTenancyIDFromCertificate(certificate)
+	fedClient, err := newX509FederationClientWithCerts(region, tenancyID, leafCertificate, leafPassphrase, leafPrivateKey, intermediateCertificates)
+	if err != nil {
+		return nil, err
+	}
+
+	provider := instancePrincipalConfigurationProvider{
+		keyProvider: instancePrincipalKeyProvider{
+			Region:           region,
+			FederationClient: fedClient,
+			TenancyID:        tenancyID,
+		},
+		region: &region,
+	}
+	return provider, nil
+
 }
 
 func (p instancePrincipalConfigurationProvider) PrivateRSAKey() (*rsa.PrivateKey, error) {

common/auth/federation_client.go

@@ -46,6 +46,22 @@ func newX509FederationClient(region common.Region, tenancyID string, leafCertifi
 	return client
 }
 
+func newX509FederationClientWithCerts(region common.Region, tenancyID string, leafCertificate, leafPassphrase, leafPrivateKey []byte, intermediateCertificates [][]byte) (federationClient, error) {
+	intermediateRetrievers := make([]x509CertificateRetriever, len(intermediateCertificates))
+	for i, c := range intermediateCertificates {
+		intermediateRetrievers[i] = &staticCertificateRetriever{Passphrase: []byte(""), CertificatePem: c, PrivateKeyPem: nil}
+	}
+
+	client := x509FederationClient{
+		tenancyID:                         tenancyID,
+		leafCertificateRetriever:          &staticCertificateRetriever{Passphrase: leafPassphrase, CertificatePem: leafCertificate, PrivateKeyPem: leafPrivateKey},
+		intermediateCertificateRetrievers: intermediateRetrievers,
+	}
+	client.sessionKeySupplier = newSessionKeySupplier()
+	client.authClient = newAuthClient(region, &client)
+	return &client, nil
+}
+
 var (
 	genericHeaders = []string{"date", "(request-target)"} // "host" is not needed for the federation endpoint.  Don't ask me why.
 	bodyHeaders    = []string{"content-length", "content-type", "x-content-sha256"}
@@ -57,7 +73,7 @@ func newAuthClient(region common.Region, provider common.KeyProvider) *common.Ba
 	if regionURL, ok := os.LookupEnv("OCI_SDK_AUTH_CLIENT_REGION_URL"); ok {
 		client.Host = regionURL
 	} else {
-		client.Host = fmt.Sprintf(common.DefaultHostURLTemplate, "auth", string(region))
+		client.Host = region.Endpoint("auth")
 	}
 	client.BasePath = "v1/x509"
 	return &client
@@ -72,7 +88,12 @@ func (c *x509FederationClient) KeyID() (string, error) {
 
 // For authClient to sign requests to X509 Federation Endpoint
 func (c *x509FederationClient) PrivateRSAKey() (*rsa.PrivateKey, error) {
-	return c.leafCertificateRetriever.PrivateKey(), nil
+	key := c.leafCertificateRetriever.PrivateKey()
+	if key == nil {
+		return nil, fmt.Errorf("can not read private key from leaf certificate. Likely an error in the metadata service")
+	}
+
+	return key, nil
 }
 
 func (c *x509FederationClient) PrivateKey() (*rsa.PrivateKey, error) {

common/auth/federation_client_test.go

@@ -8,13 +8,14 @@ import (
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
-	"github.com/oracle/oci-go-sdk/common"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/mock"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 	"testing"
+
+	"github.com/oracle/oci-go-sdk/common"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 )
 
 func TestX509FederationClient_VeryFirstSecurityToken(t *testing.T) {
@@ -304,6 +305,31 @@ func TestX509FederationClient_AuthServerInternalError(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestX509FederationClient_ClientHost(t *testing.T) {
+	type testData struct {
+		region   common.Region
+		expected string
+	}
+	testDataSet := []testData{
+		{
+			// OC1
+			region:   common.StringToRegion("us-phoenix-1"),
+			expected: "auth.us-phoenix-1.oraclecloud.com",
+		},
+		{
+			// unknown
+			region:   common.StringToRegion("test"),
+			expected: "auth.test.oraclecloud.com",
+		},
+	}
+
+	for _, testData := range testDataSet {
+		federationClient := &x509FederationClient{}
+		federationClient.authClient = newAuthClient(testData.region, federationClient)
+		assert.Equal(t, testData.expected, federationClient.authClient.Host)
+	}
+}
+
 func parseCertificate(certPem string) *x509.Certificate {
 	var block *pem.Block
 	block, _ = pem.Decode([]byte(certPem))
@@ -387,7 +413,7 @@ ysvMnQwaC0432ceRJ3r6vPAI2EPRd9KOE7Va1IFNJNmOuIkmRx8t`
 	//				"opc-certtype:instance",
 	//				"opc-instance.ocid1.phx.bluhbluhbluh",
 	//				"opc-compartment:ocid1.compartment.oc1.bluhbluhbluh",
-	//				fmt.Sprintf("opc-tenant:%s", tenancyID),
+	//				fmt.Sprintf("opc-tenant:%s", TenancyID),
 	//			},
 	//		},
 	//		NotBefore:          notBefore,