From 59f267808d02d93d61f3835b6740201b8d4d58b7 Mon Sep 17 00:00:00 2001 From: Ilya Kislenko Date: Thu, 11 Jul 2019 13:57:37 -0700 Subject: [PATCH] Upgrading ibm libs and enabling ibm file (#6007) * Updating ibm libs to the latest * ibm libs renamed all function names refactoring our code with new function names also modifing client default configs * removing Skips, Renaming functions * removing Skips and changing correct tets config * actually remocing Skips and comments * one more rename * Enabling IBMFile Tests * Enabling ibm file (#6011) * adding SLFile into kio getter * Enabling ibm-file snapshots * Adding SoftlayerFile type into blockstore * adding Authorized Hosts to volume * Post merge cleanups --- pkg/blockstorage/ibm/client.go | 13 +++-- pkg/blockstorage/ibm/client_kube_test.go | 6 +-- pkg/blockstorage/ibm/client_test.go | 6 +-- pkg/blockstorage/ibm/ibmcloud.go | 48 ++++++++++++------- pkg/blockstorage/ibm/ibmcloud_test.go | 10 ++-- .../ibm/testdata/correct/libconfig.toml | 21 ++++++-- pkg/blockstorage/ibm/utils/ibmcloud_utils.go | 42 ++++++++++++++++ 7 files changed, 106 insertions(+), 40 deletions(-) create mode 100644 pkg/blockstorage/ibm/utils/ibmcloud_utils.go diff --git a/pkg/blockstorage/ibm/client.go b/pkg/blockstorage/ibm/client.go index e82a5fdf40..39dd42c9ff 100644 --- a/pkg/blockstorage/ibm/client.go +++ b/pkg/blockstorage/ibm/client.go @@ -47,6 +47,11 @@ var ( SoftlayerVolProvisionTimeout: "10m", SoftlayerRetryInterval: "5s", } + + vpcCfg = ibmcfg.VPCProviderConfig{ + Enabled: false, + VPCBlockProviderName: "vpc-classic", + } ) //client is a wrapper for Library client @@ -66,14 +71,11 @@ func newClient(ctx context.Context, args map[string]string) (*client, error) { return nil, errors.New("Failed to get IBM client config") } - provName := softLayerCfg.SoftlayerBlockProviderName + provName := cfg.Softlayer.SoftlayerBlockProviderName if enableFile, ok := args[SoftlayerFileAttName]; ok && enableFile == "true" { cfg.Softlayer.SoftlayerFileEnabled = true cfg.Softlayer.SoftlayerBlockEnabled = false - provName = softLayerCfg.SoftlayerFileProviderName - } else { - cfg.Softlayer.SoftlayerFileEnabled = false - cfg.Softlayer.SoftlayerBlockEnabled = true + provName = cfg.Softlayer.SoftlayerFileProviderName } provReg, err := ibmprovutils.InitProviders(cfg, zaplog) @@ -101,6 +103,7 @@ func findDefaultConfig(ctx context.Context, args map[string]string, zaplog *zap. Softlayer: &softLayerCfg, Gen2: &ibmcfg.Gen2Config{}, Bluemix: &blueMixCfg, + VPC: &vpcCfg, }, nil } // Cheking if IBM store secret is present diff --git a/pkg/blockstorage/ibm/client_kube_test.go b/pkg/blockstorage/ibm/client_kube_test.go index 8aa0dd6386..5658421942 100644 --- a/pkg/blockstorage/ibm/client_kube_test.go +++ b/pkg/blockstorage/ibm/client_kube_test.go @@ -27,8 +27,6 @@ type KubeTestIBMClient struct { var _ = Suite(&KubeTestIBMClient{}) func (s *KubeTestIBMClient) SetUpSuite(c *C) { - // TODO: IBM client changed internally, unskip after K10-2454 - c.Skip("Skipped until we fix K10-2454") var secData []byte var err error if tomlPath, ok := os.LookupEnv(workAroundEnv); ok { @@ -55,8 +53,6 @@ func (s *KubeTestIBMClient) SetUpSuite(c *C) { } func (s KubeTestIBMClient) TearDownSuite(c *C) { - // TODO: IBM client changed internally, unskip after K10-2454 - c.Skip("Skipped until we fix K10-2454") if _, ok := os.LookupEnv(workAroundEnv); !ok { c.Skip(workAroundEnv + "TOML path is not present") } @@ -75,6 +71,6 @@ func (s KubeTestIBMClient) TestIBMSecret(c *C) { c.Assert(ibmCli.Service, NotNil) defer ibmCli.Service.Close() c.Assert(*ibmCli, FitsTypeOf, client{}) - _, err = ibmCli.Service.SnapshotsList() + _, err = ibmCli.Service.ListSnapshots() c.Assert(err, IsNil) } diff --git a/pkg/blockstorage/ibm/client_test.go b/pkg/blockstorage/ibm/client_test.go index 3b280a2e4d..5710b31545 100644 --- a/pkg/blockstorage/ibm/client_test.go +++ b/pkg/blockstorage/ibm/client_test.go @@ -30,8 +30,6 @@ type ClientSuite struct { var _ = Suite(&ClientSuite{}) func (s *ClientSuite) SetUpSuite(c *C) { - // TODO: IBM client changed internally, unskip after K10-2454 - c.Skip("Skipped until we fix K10-2454") if os.Getenv(IBMApiKeyEnv) == "" { c.Skip(IBMApiKeyEnv + " envionment variable not set") } @@ -56,7 +54,7 @@ func (s *ClientSuite) TestAPIClient(c *C) { c.Assert(ibmCli.Service, NotNil) defer ibmCli.Service.Close() c.Assert(*ibmCli, FitsTypeOf, client{}) - _, err = ibmCli.Service.SnapshotsList() + _, err = ibmCli.Service.ListSnapshots() c.Assert(err, IsNil) } @@ -74,7 +72,7 @@ func (s *ClientSuite) TestIBMClientSoftlayerFile(c *C) { c.Assert(*ibmCli, FitsTypeOf, client{}) c.Assert(ibmCli.SLCfg.SoftlayerBlockEnabled, Equals, false) c.Assert(ibmCli.SLCfg.SoftlayerFileEnabled, Equals, true) - _, err = ibmCli.Service.SnapshotsList() + _, err = ibmCli.Service.ListSnapshots() c.Assert(err, IsNil) } diff --git a/pkg/blockstorage/ibm/ibmcloud.go b/pkg/blockstorage/ibm/ibmcloud.go index a9dd5c200a..1d163aa57e 100644 --- a/pkg/blockstorage/ibm/ibmcloud.go +++ b/pkg/blockstorage/ibm/ibmcloud.go @@ -16,6 +16,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/kanisterio/kanister/pkg/blockstorage" + bsibmutils "github.com/kanisterio/kanister/pkg/blockstorage/ibm/utils" ktags "github.com/kanisterio/kanister/pkg/blockstorage/tags" ) @@ -59,7 +60,7 @@ func (s *ibmCloud) VolumeCreate(ctx context.Context, volume blockstorage.Volume) newVol.VolumeNotes = blockstorage.KeyValueToMap(volume.Tags) newVol.Az = s.cli.SLCfg.SoftlayerDataCenter log.Debugf("Creating new volume %+v", newVol) - volR, err := s.cli.Service.VolumeCreate(newVol) + volR, err := s.cli.Service.CreateVolume(newVol) if err != nil { return nil, errors.New(fmt.Sprintf("Failed to create volume. %s", err.Error())) } @@ -68,7 +69,7 @@ func (s *ibmCloud) VolumeCreate(ctx context.Context, volume blockstorage.Volume) } func (s *ibmCloud) VolumeGet(ctx context.Context, id string, zone string) (*blockstorage.Volume, error) { - vol, err := s.cli.Service.VolumeGet(id) + vol, err := s.cli.Service.GetVolume(id) if err != nil { return nil, errors.Wrapf(err, "Failed to get volume with id %s", id) } @@ -108,10 +109,16 @@ func (s *ibmCloud) volumeParse(ctx context.Context, vol *ibmprov.Volume) (*block } attribs[LunIDAttName] = vol.LunID - if len(vol.TargetIPAddresses) < 0 { - return nil, errors.New("TargetIPAddresses are missing from Volume info") + if len(vol.IscsiTargetIPAddresses) < 0 { + return nil, errors.New("IscsiTargetIPAddresses are missing from Volume info") } - attribs[TargetIPsAttName] = strings.Join(vol.TargetIPAddresses, ",") + if len(vol.Attributes) > 0 { + for k, v := range vol.Attributes { + attribs[k] = v + } + } + + attribs[TargetIPsAttName] = strings.Join(vol.IscsiTargetIPAddresses, ",") return &blockstorage.Volume{ Type: s.Type(), @@ -129,7 +136,7 @@ func (s *ibmCloud) volumeParse(ctx context.Context, vol *ibmprov.Volume) (*block func (s *ibmCloud) VolumesList(ctx context.Context, tags map[string]string, zone string) ([]*blockstorage.Volume, error) { var vols []*blockstorage.Volume - ibmVols, err := s.cli.Service.VolumesList(tags) + ibmVols, err := s.cli.Service.ListVolumes(tags) if err != nil { return nil, errors.Wrapf(err, "Failed to list volumes with tags %v", tags) } @@ -176,7 +183,7 @@ func (s *ibmCloud) SnapshotsList(ctx context.Context, tags map[string]string) ([ var snaps []*blockstorage.Snapshot // IBM doens't support tag for snapshot list. // Getting all of them - ibmsnaps, err := s.cli.Service.SnapshotsList() + ibmsnaps, err := s.cli.Service.ListSnapshots() if err != nil { return nil, errors.Wrap(err, "Failed to list Snapshots") } @@ -195,7 +202,7 @@ func (s *ibmCloud) SnapshotCopy(ctx context.Context, from, to blockstorage.Snaps func (s *ibmCloud) SnapshotCreate(ctx context.Context, volume blockstorage.Volume, tags map[string]string) (*blockstorage.Snapshot, error) { alltags := ktags.GetTags(tags) - ibmvol, err := s.cli.Service.VolumeGet(volume.ID) + ibmvol, err := s.cli.Service.GetVolume(volume.ID) if err != nil { return nil, errors.Wrapf(err, "Failed to get Volume for Snapshot creation, volume_id :%s", volume.ID) } @@ -203,7 +210,7 @@ func (s *ibmCloud) SnapshotCreate(ctx context.Context, volume blockstorage.Volum if ibmvol.SnapshotSpace == nil { log.Debugf("Ordering snapshot space for volume %+v", ibmvol) ibmvol.SnapshotSpace = ibmvol.Capacity - err = s.cli.Service.SnapshotOrder(*ibmvol) + err = s.cli.Service.OrderSnapshot(*ibmvol) if err != nil { if strings.Contains(err.Error(), "already has snapshot space") != true { return nil, errors.Wrapf(err, "Failed to order Snapshot space, volume_id :%s", volume.ID) @@ -217,7 +224,7 @@ func (s *ibmCloud) SnapshotCreate(ctx context.Context, volume blockstorage.Volum } } log.Debugf("Creating snapshot for vol %+v", ibmvol) - snap, err := s.cli.Service.SnapshotCreate(ibmvol, alltags) + snap, err := s.cli.Service.CreateSnapshot(ibmvol, alltags) if err != nil { return nil, errors.Wrapf(err, "Failed to create snapshot, volume_id: %s", volume.ID) } @@ -231,15 +238,15 @@ func (s *ibmCloud) SnapshotCreateWaitForCompletion(ctx context.Context, snap *bl } func (s *ibmCloud) SnapshotDelete(ctx context.Context, snapshot *blockstorage.Snapshot) error { - snap, err := s.cli.Service.SnapshotGet(snapshot.ID) + snap, err := s.cli.Service.GetSnapshot(snapshot.ID) if err != nil { return errors.Wrapf(err, "Failed to get Snapshot for deletion, snapshot_id %s", snapshot.ID) } - return s.cli.Service.SnapshotDelete(snap) + return s.cli.Service.DeleteSnapshot(snap) } func (s *ibmCloud) SnapshotGet(ctx context.Context, id string) (*blockstorage.Snapshot, error) { - snap, err := s.cli.Service.SnapshotGet(id) + snap, err := s.cli.Service.GetSnapshot(id) if err != nil { return nil, errors.Wrapf(err, "Failed to get Snapshot, snapshot_id %s", id) } @@ -247,11 +254,11 @@ func (s *ibmCloud) SnapshotGet(ctx context.Context, id string) (*blockstorage.Sn } func (s *ibmCloud) VolumeDelete(ctx context.Context, volume *blockstorage.Volume) error { - ibmvol, err := s.cli.Service.VolumeGet(volume.ID) + ibmvol, err := s.cli.Service.GetVolume(volume.ID) if err != nil { return errors.Wrapf(err, "Failed to get Volume for deletion, volume_id :%s", volume.ID) } - return s.cli.Service.VolumeDelete(ibmvol) + return s.cli.Service.DeleteVolume(ibmvol) } func (s *ibmCloud) SetTags(ctx context.Context, resource interface{}, tags map[string]string) error { @@ -270,7 +277,7 @@ func (s *ibmCloud) VolumeCreateFromSnapshot(ctx context.Context, snapshot blocks tags[tag.Key] = tag.Value } } - snap, err := s.cli.Service.SnapshotGet(snapshot.ID) + snap, err := s.cli.Service.GetSnapshot(snapshot.ID) if err != nil { return nil, errors.Wrapf(err, "Failed to get Snapshot for Volume Restore, snapshot_id %s", snapshot.ID) } @@ -280,10 +287,15 @@ func (s *ibmCloud) VolumeCreateFromSnapshot(ctx context.Context, snapshot blocks snap.Volume.VolumeID = snapshot.Volume.ID log.Debugf("Snapshot with new volume ID %+v", snap) - vol, err := s.cli.Service.VolumeCreateFromSnapshot(*snap, tags) + vol, err := s.cli.Service.CreateVolumeFromSnapshot(*snap, tags) if err != nil { return nil, errors.Wrapf(err, "Failed to create Volume from Snapshot, snapshot_id %s", snapshot.ID) } + + if err = bsibmutils.AuthorizeSoftLayerFileHosts(ctx, vol, s.cli.Service); err != nil { + return nil, errors.Wrap(err, "Failed to add Authorized Hosts to new volume, without Authorized Hosts ibm storage will not be able to mount volume to kubernetes node") + } + return s.VolumeGet(ctx, vol.VolumeID, snapshot.Volume.Az) } @@ -295,7 +307,7 @@ func waitforSnapSpaceOrder(ctx context.Context, cli *client, id string) error { Max: 10 * time.Second, } return poll.WaitWithBackoff(ctx, snapWaitBackoff, func(ctx context.Context) (bool, error) { - vol, err := cli.Service.VolumeGet(id) + vol, err := cli.Service.GetVolume(id) if err != nil { return false, errors.Wrapf(err, "Failed to get volume, volume_id: %s", id) } diff --git a/pkg/blockstorage/ibm/ibmcloud_test.go b/pkg/blockstorage/ibm/ibmcloud_test.go index 6bff0d550a..7fddde2b0b 100644 --- a/pkg/blockstorage/ibm/ibmcloud_test.go +++ b/pkg/blockstorage/ibm/ibmcloud_test.go @@ -10,6 +10,7 @@ import ( "time" ibmprov "github.com/IBM/ibmcloud-storage-volume-lib/lib/provider" + "github.com/luci/go-render/render" . "gopkg.in/check.v1" "github.com/kanisterio/kanister/pkg/blockstorage" @@ -28,15 +29,14 @@ type TestIBMCloud struct { volAtts map[string]string } +//These are not executed as part of Pipeline, but usefull for development var softlayerVolAtts = map[string]string{ ProviderTypeAttName: string(ibmprov.VolumeProviderType("endurance")), TierAttName: "2", } var _ = Suite(&TestIBMCloud{softlayerFile: false, volAtts: softlayerVolAtts}) - -// Commented out untill IBM fixes SL on their side. -// var _ = Suite(&TestIBMCloud{softlayerFile: true, volAtts: softlayerVolAtts}) +var _ = Suite(&TestIBMCloud{softlayerFile: true, volAtts: softlayerVolAtts}) func (s *TestIBMCloud) SetUpSuite(c *C) { c.Skip("IBM tests are too flaky to run in CI") @@ -50,6 +50,7 @@ func (s *TestIBMCloud) SetUpSuite(c *C) { if s.softlayerFile { args[SoftlayerFileAttName] = "true" } + var err error ctx := context.Background() s.provider, err = NewProvider(ctx, args) @@ -69,6 +70,7 @@ func (s *TestIBMCloud) SetUpSuite(c *C) { } tmpVol.Az = s.cli.SLCfg.SoftlayerDataCenter s.testVol, err = s.provider.VolumeCreate(ctx, *tmpVol) + c.Log(fmt.Sprintf("sl cfg %s", render.Render(softLayerCfg))) c.Assert(err, IsNil) c.Assert(s.testVol.ID, NotNil) } @@ -120,7 +122,7 @@ func (s TestIBMCloud) TestVolRestore(c *C) { tTags := map[string]string{"ibmblock_unit_test_restore_vol": fmt.Sprintf("test-vol-%d", time.Now().Unix())} resVol, err := s.provider.VolumeCreateFromSnapshot(context.Background(), *bsSnap, tTags) c.Assert(err, IsNil) - cVol, err := s.cli.Service.VolumeGet(resVol.ID) + cVol, err := s.cli.Service.GetVolume(resVol.ID) c.Assert(err, IsNil) c.Assert(cVol, NotNil) err = s.provider.VolumeDelete(context.Background(), resVol) diff --git a/pkg/blockstorage/ibm/testdata/correct/libconfig.toml b/pkg/blockstorage/ibm/testdata/correct/libconfig.toml index 14d4b6ba9d..ed335af683 100644 --- a/pkg/blockstorage/ibm/testdata/correct/libconfig.toml +++ b/pkg/blockstorage/ibm/testdata/correct/libconfig.toml @@ -1,10 +1,13 @@ +[server] +debug_trace = false + [bluemix] -iam_url = "https://iam.stage1.ng.bluemix.net" +iam_url = "https://iam.bluemix.net" iam_client_id = "bx" iam_client_secret = "bx" iam_api_key = "free" refresh_token = "" - +containers_api_route = "https://origin.containers.test.cloud.ibm.com" [softlayer] softlayer_block_enabled = true @@ -15,11 +18,21 @@ softlayer_username = "" softlayer_api_key = "" softlayer_endpoint_url = "https://api.softlayer.com/rest/v3" softlayer_iam_endpoint_url = "https://api.softlayer.com/mobile/v3" -softlayer_datacenter = "mex01" -softlayer_api_timeout = "20s" +softlayer_datacenter = "sjc03" +softlayer_api_timeout = "30s" +softlayer_vol_provision_timeout = "30m" +softlayer_api_retry_interval = "5s" [genesis] genesis_provider_enabled = false genesis_user_name = "" genesis_api_key = "" genesis_url = "" + +[vpc] +vpc_enabled = false +vpc_block_provider_name = "vpc-classic" + +[iks] +iks_enabled = false +iks_block_provider_name = "iks-vpc-classic" diff --git a/pkg/blockstorage/ibm/utils/ibmcloud_utils.go b/pkg/blockstorage/ibm/utils/ibmcloud_utils.go new file mode 100644 index 0000000000..8e5eaaa090 --- /dev/null +++ b/pkg/blockstorage/ibm/utils/ibmcloud_utils.go @@ -0,0 +1,42 @@ +package ibmutils + +// IBM Cloud utils + +import ( + "context" + + ibmprov "github.com/IBM/ibmcloud-storage-volume-lib/lib/provider" + "github.com/pkg/errors" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kanisterio/kanister/pkg/kube" +) + +// AuthorizeSoftLayerFileHosts some of volumes are required post creation authorization to be mounted +func AuthorizeSoftLayerFileHosts(ctx context.Context, vol *ibmprov.Volume, slCli ibmprov.Session) error { + k8scli, err := kube.NewClient() + if err != nil { + return errors.Wrap(err, "Failed to created k8s client.") + } + + nodes, err := k8scli.CoreV1().Nodes().List(metav1.ListOptions{}) + if err != nil { + return errors.Wrap(err, "Failed to list nodes") + } + + nodeips := []string{} + + for _, node := range nodes.Items { + for _, ip := range node.Status.Addresses { + if ip.Type == v1.NodeInternalIP { + nodeips = append(nodeips, ip.Address) + } + } + } + + return slCli.AuthorizeVolume(ibmprov.VolumeAuthorization{ + Volume: *vol, + HostIPs: nodeips, + }) +}