Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(frontend): target api support #314

Merged
merged 4 commits into from
Oct 3, 2023
Merged

feat(frontend): target api support #314

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d8035a2
chore(deps): udpate opi-spdk-bridge to 3e4c52f
artek-koltun Oct 3, 2023
ceecdc0
feat(frontend): add frontend target api support
artek-koltun Oct 3, 2023
016658a
test(frontend): frontend target api update
artek-koltun Oct 3, 2023
6980c08
docs: add nvme tcp controller calls
artek-koltun Oct 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,13 +107,18 @@ or specify commands manually
grpc_cli call --json_input --json_output $BRIDGE_ADDR CreateNvmeSubsystem "{nvme_subsystem : {spec : {nqn: 'nqn.2022-09.io.spdk:opitest2', serial_number: 'myserial2', model_number: 'mymodel2', max_namespaces: 11} }, nvme_subsystem_id : 'subsystem2' }"
grpc_cli call --json_input --json_output $BRIDGE_ADDR ListNvmeSubsystems "{}"
grpc_cli call --json_input --json_output $BRIDGE_ADDR GetNvmeSubsystem "{name : '//storage.opiproject.org/subsystems/subsystem2'}"
grpc_cli call --json_input --json_output $BRIDGE_ADDR CreateNvmeController "{parent: '//storage.opiproject.org/subsystems/subsystem2', nvme_controller : {spec : {nvme_controller_id: 2, pcie_id : {physical_function : 0, virtual_function : 0, port_id: 0}, max_nsq:5, max_ncq:5 } }, nvme_controller_id : 'controller1'}"
grpc_cli call --json_input --json_output $BRIDGE_ADDR CreateNvmeController "{parent: '//storage.opiproject.org/subsystems/subsystem2', nvme_controller : {spec : {nvme_controller_id: 2, pcie_id : {physical_function : 0, virtual_function : 0, port_id: 0}, max_nsq:5, max_ncq:5, 'trtype': 'NVME_TRANSPORT_PCIE' } }, nvme_controller_id : 'controller1'}"
grpc_cli call --json_input --json_output $BRIDGE_ADDR ListNvmeControllers "{parent : '//storage.opiproject.org/subsystems/subsystem2'}"
grpc_cli call --json_input --json_output $BRIDGE_ADDR GetNvmeController "{name : '//storage.opiproject.org/subsystems/subsystem2/controllers/controller1'}"

# Nvme VF creation on PF0
grpc_cli call --json_input --json_output $BRIDGE_ADDR CreateNvmeSubsystem "{nvme_subsystem : {spec : {nqn: 'nqn.2022-09.io.spdk:opitest3', serial_number: 'mev-opi-serial', model_number: 'mev-opi-model', max_namespaces: 11} }, nvme_subsystem_id : 'subsystem03' }"
grpc_cli call --json_input --json_output $BRIDGE_ADDR CreateNvmeController "{parent: '//storage.opiproject.org/subsystems/subsystem03', nvme_controller : {spec : {nvme_controller_id: 2, pcie_id : {physical_function : 0, virtual_function : 3, port_id: 0 }, max_nsq:5, max_ncq:5 } }, nvme_controller_id : 'controller3'}"
grpc_cli call --json_input --json_output $BRIDGE_ADDR CreateNvmeController "{parent: '//storage.opiproject.org/subsystems/subsystem03', nvme_controller : {spec : {nvme_controller_id: 2, pcie_id : {physical_function : 0, virtual_function : 3, port_id: 0 }, max_nsq:5, max_ncq:5, 'trtype': 'NVME_TRANSPORT_PCIE' } }, nvme_controller_id : 'controller3'}"

# Create Nvme/TCP controller
grpc_cli call --json_input --json_output $BRIDGE_ADDR CreateNvmeSubsystem "{nvme_subsystem : {spec : {nqn: 'nqn.2022-09.io.spdk:opitest4', serial_number: 'myserial2', model_number: 'mymodel2', max_namespaces: 11} }, nvme_subsystem_id : 'subsystem4' }"
grpc_cli call --json_input --json_output $BRIDGE_ADDR CreateNvmeController "{'parent':'//storage.opiproject.org/subsystems/subsystem4','nvme_controller':{'spec':{'nvme_controller_id':2,'fabrics_id':{'traddr': '127.0.0.1', trsvcid: '4421', adrfam: 'NVME_ADRFAM_IPV4'}, 'max_nsq':5,'max_ncq':5, 'trtype': 'NVME_TRANSPORT_TCP'}},'nvme_controller_id':'controller4'}"
grpc_cli call --json_input --json_output $BRIDGE_ADDR GetNvmeController "{name : '//storage.opiproject.org/subsystems/subsystem4/controllers/controller4'}"

# Connect to remote storage-target
grpc_cli call --json_input --json_output $BRIDGE_ADDR CreateNvmeRemoteController "{nvme_remote_controller : {multipath: 'NVME_MULTIPATH_MULTIPATH'}, nvme_remote_controller_id: 'nvmetcp12'}"
Expand Down Expand Up @@ -169,6 +174,10 @@ grpc_cli call --json_input --json_output $BRIDGE_ADDR DeleteNvmeSubsystem "{name
# Delete Nvme PF
grpc_cli call --json_input --json_output $BRIDGE_ADDR DeleteNvmeController "{name : '//storage.opiproject.org/subsystems/subsystem2/controllers/controller1'}"
grpc_cli call --json_input --json_output $BRIDGE_ADDR DeleteNvmeSubsystem "{name : '//storage.opiproject.org/subsystems/subsystem2'}"

# Delete Nvme/TCP
grpc_cli call --json_input --json_output $BRIDGE_ADDR DeleteNvmeController "{name : '//storage.opiproject.org/subsystems/subsystem4/controllers/controller4'}"
grpc_cli call --json_input --json_output $BRIDGE_ADDR DeleteNvmeSubsystem "{name : '//storage.opiproject.org/subsystems/subsystem4'}"
```

To observe Nvme devices on host:
Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ require (
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0
github.com/onsi/ginkgo/v2 v2.12.1
github.com/opiproject/gospdk v0.0.0-20230924031820-4b65d0851cb9
github.com/opiproject/opi-api v0.0.0-20230926113732-f307f1aa0bd1
github.com/opiproject/opi-api v0.0.0-20231002080217-fc749d07fdda
github.com/opiproject/opi-smbios-bridge v0.1.3-0.20230930033102-4953908f3ea6
github.com/opiproject/opi-spdk-bridge v0.1.2-0.20230930110652-ceaa970c6018
github.com/opiproject/opi-spdk-bridge v0.1.2-0.20231002185927-3e4c52f5674a
github.com/opiproject/opi-strongswan-bridge v0.1.1
github.com/philippgille/gokv v0.6.0
github.com/philippgille/gokv/gomap v0.6.0
Expand Down
17 changes: 5 additions & 12 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -426,18 +426,12 @@ github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xl
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
github.com/opiproject/gospdk v0.0.0-20230924031820-4b65d0851cb9 h1:DkqXBfcVaJEtsyziW8ZW/Vwge5H3Skk0ZfVm5D/s0LM=
github.com/opiproject/gospdk v0.0.0-20230924031820-4b65d0851cb9/go.mod h1:rO2nM/OyY69zPLQO2mNX8+ckvyAd2TqV1vLLG7qEB74=
github.com/opiproject/opi-api v0.0.0-20230924031502-67b3821b62cf h1:i7kajMeK+FCJzVPEF5NXvc1y3joJLOz71/rNxtu7IMo=
github.com/opiproject/opi-api v0.0.0-20230924031502-67b3821b62cf/go.mod h1:92pv4ulvvPMuxCJ9ND3aYbmBfEMLx0VCjpkiR7ZTqPY=
github.com/opiproject/opi-api v0.0.0-20230926113732-f307f1aa0bd1 h1:BG8LBieb6Fomf2IBoodUE1QUauf5Cn8XWtqU2rK7NIE=
github.com/opiproject/opi-api v0.0.0-20230926113732-f307f1aa0bd1/go.mod h1:92pv4ulvvPMuxCJ9ND3aYbmBfEMLx0VCjpkiR7ZTqPY=
github.com/opiproject/opi-smbios-bridge v0.1.3-0.20230924050511-ed2cf72c2fbf h1:cU3THJncZF2RCQI79EIwHtTnKm2V8A4IM0EeZYI34sk=
github.com/opiproject/opi-smbios-bridge v0.1.3-0.20230924050511-ed2cf72c2fbf/go.mod h1:1x0zzecRFH8wHC9Uuv1P8Qr1pfEaYCYuJZ2qDjzJchM=
github.com/opiproject/opi-api v0.0.0-20231002080217-fc749d07fdda h1:w/HTrqmBrXabdcc71ACkRQqavKDFvfPp0MqiGWKTGjw=
github.com/opiproject/opi-api v0.0.0-20231002080217-fc749d07fdda/go.mod h1:92pv4ulvvPMuxCJ9ND3aYbmBfEMLx0VCjpkiR7ZTqPY=
github.com/opiproject/opi-smbios-bridge v0.1.3-0.20230930033102-4953908f3ea6 h1:0TCR2UmJcQTCCH1uK0LdZ6LcBycO3uUAnLwfBqs6rIM=
github.com/opiproject/opi-smbios-bridge v0.1.3-0.20230930033102-4953908f3ea6/go.mod h1:qWMKMtmpgqXDoV4T+VYY6w9vwreU2y14eY9y5pRpsW8=
github.com/opiproject/opi-spdk-bridge v0.1.2-0.20230928235811-ca9c73f02de8 h1:4moAe2QbiAflam0NoPzhlguh2hTPQP/Nb34vV+Bq1h4=
github.com/opiproject/opi-spdk-bridge v0.1.2-0.20230928235811-ca9c73f02de8/go.mod h1:UAuX2B9G5kwtYhQ25VZkXX5ey+UINOSFIQOBkp9f7J0=
github.com/opiproject/opi-spdk-bridge v0.1.2-0.20230930110652-ceaa970c6018 h1:hH4L5mPMY7LZ/vwKXRH2S7GE+Qdt7Kond7OoAFbkRYY=
github.com/opiproject/opi-spdk-bridge v0.1.2-0.20230930110652-ceaa970c6018/go.mod h1:salrjuR7iOU6ui8biDhZN1JZeXoao7DMLlnjMS8ID/o=
github.com/opiproject/opi-spdk-bridge v0.1.2-0.20231002185927-3e4c52f5674a h1:CsPCuNznK/rmCajGVnQxCAOOORPIHhoqW7B3dn2chAM=
github.com/opiproject/opi-spdk-bridge v0.1.2-0.20231002185927-3e4c52f5674a/go.mod h1:ZrCM2nnXV7jVlxdKRop8+U83Az13qG9G29RqUSRcl8k=
github.com/opiproject/opi-strongswan-bridge v0.1.1 h1:Mz/8AtA0DD8O/H9jCsDw7wuVoWNiqgXICLsD10XWJ+g=
github.com/opiproject/opi-strongswan-bridge v0.1.1/go.mod h1:ek3r3zLa9nOfb2a7ybdMJvb5BSGU9I17Xo38UMXfE+k=
github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k=
Expand Down Expand Up @@ -595,8 +589,7 @@ github.com/ultraware/whitespace v0.0.5 h1:hh+/cpIcopyMYbZNVov9iSxvJU3OYQg78Sfaqz
github.com/ultraware/whitespace v0.0.5/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
github.com/uudashr/gocognit v1.0.7 h1:e9aFXgKgUJrQ5+bs61zBigmj7bFJ/5cC6HmMahVzuDo=
github.com/uudashr/gocognit v1.0.7/go.mod h1:nAIUuVBnYU7pcninia3BHOvQkpQCeO76Uscky5BOwcY=
github.com/vektra/mockery/v2 v2.34.0 h1:guzK+ZqivTzSNqp3EwM/tKiRQlkMpkMQorYXpXZ2XAs=
github.com/vektra/mockery/v2 v2.34.0/go.mod h1:9lREs4VEeQiUS3rizYQx1saxHu2JiIhThP0q9+fDegM=
github.com/vektra/mockery/v2 v2.34.2 h1:sB9pOos4PXHxznWgj7T+LQQJr4rni7MJaWgsw6JaOFI=
github.com/vektra/mockery/v2 v2.34.2/go.mod h1:9lREs4VEeQiUS3rizYQx1saxHu2JiIhThP0q9+fDegM=
github.com/xen0n/gosmopolitan v1.2.1 h1:3pttnTuFumELBRSh+KQs1zcz4fN6Zy7aB0xlnQSn1Iw=
github.com/xen0n/gosmopolitan v1.2.1/go.mod h1:JsHq/Brs1o050OOdmzHeOr0N7OtlnKRAGAsElF8xBQA=
Expand Down
7 changes: 6 additions & 1 deletion pkg/frontend/frontend.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,12 @@ func NewServer(jsonRPC spdk.JSONRPC, store gokv.Store) *Server {
log.Panic("nil for Store is not allowed")
}
opiSpdkServer := frontend.NewCustomizedServer(
jsonRPC, store, NewNvmeNpiTransport(), NewMevBlkTransport())
jsonRPC, store,
map[pb.NvmeTransportType]frontend.NvmeTransport{
pb.NvmeTransportType_NVME_TRANSPORT_PCIE: NewNvmeNpiTransport(),
pb.NvmeTransportType_NVME_TRANSPORT_TCP: frontend.NewNvmeTCPTransport(),
},
NewMevBlkTransport())
return &Server{
opiSpdkServer,
opiSpdkServer,
Expand Down
11 changes: 5 additions & 6 deletions pkg/frontend/nvme.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,20 +29,20 @@ func NewNvmeNpiTransport() frontend.NvmeTransport {
}

func (c nvmeNpiTransport) Params(ctrlr *pb.NvmeController, nqn string) (spdk.NvmfSubsystemAddListenerParams, error) {
if ctrlr.Spec.PcieId.PortId.Value != 0 {
if ctrlr.GetSpec().GetPcieId().GetPortId().GetValue() != 0 {
return spdk.NvmfSubsystemAddListenerParams{},
errors.New("only port 0 is supported")
}

if ctrlr.Spec.PcieId.PhysicalFunction.Value != 0 {
if ctrlr.GetSpec().GetPcieId().GetPhysicalFunction().GetValue() != 0 {
return spdk.NvmfSubsystemAddListenerParams{},
errors.New("only physical_function 0 is supported")
}

result := spdk.NvmfSubsystemAddListenerParams{}
result.Nqn = nqn
result.ListenAddress.Trtype = "npi"
result.ListenAddress.Traddr = calculateTransportAddr(ctrlr.Spec.PcieId)
result.ListenAddress.Traddr = calculateTransportAddr(ctrlr.GetSpec().GetPcieId())
return result, nil
}

Expand All @@ -59,7 +59,7 @@ func (s *Server) CreateNvmeController(ctx context.Context, in *pb.CreateNvmeCont

log.Printf("Passing request to opi-spdk-bridge")
response, err := s.FrontendNvmeServiceServer.CreateNvmeController(ctx, in)
if err == nil {
if err == nil && in.GetNvmeController().GetSpec().GetTrtype() == pb.NvmeTransportType_NVME_TRANSPORT_PCIE {
// response contains different QoS limits. It is an indication that
// opi-spdk-bridge returned an already existing controller providing idempotence
if !proto.Equal(response.Spec.MaxLimit, in.NvmeController.Spec.MaxLimit) ||
Expand Down Expand Up @@ -88,8 +88,7 @@ func (s *Server) UpdateNvmeController(ctx context.Context, in *pb.UpdateNvmeCont
originalNvmeController := s.nvme.Controllers[in.NvmeController.Name]
log.Printf("Passing request to opi-spdk-bridge")
response, err := s.FrontendNvmeServiceServer.UpdateNvmeController(ctx, in)

if err == nil {
if err == nil && in.GetNvmeController().GetSpec().GetTrtype() == pb.NvmeTransportType_NVME_TRANSPORT_PCIE {
if qosErr := s.setNvmeQosLimit(in.NvmeController); qosErr != nil {
log.Println("Failed to set qos settings:", qosErr)
log.Println("Restore original controller")
Expand Down
Loading
Loading