One document matched: draft-ietf-nfsv4-rfc5666bis-04.ps
%!PS-Adobe-3.0
%%BoundingBox: 76 24 571 766
%%Title: Enscript Output
%%For: Chuck Lever
%%Creator: GNU Enscript 1.6.6
%%CreationDate: Fri Mar 4 07:58:09 2016
%%Orientation: Portrait
%%Pages: (atend)
%%DocumentMedia: A4 595 842 0 () ()
%%DocumentNeededResources: (atend)
%%EndComments
%%BeginProlog
%%BeginResource: procset Enscript-Prolog 1.6 6
%
% Procedures.
%
/_S { % save current state
/_s save def
} def
/_R { % restore from saved state
_s restore
} def
/S { % showpage protecting gstate
gsave
showpage
grestore
} bind def
/MF { % fontname newfontname -> - make a new encoded font
/newfontname exch def
/fontname exch def
/fontdict fontname findfont def
/newfont fontdict maxlength dict def
fontdict {
exch
dup /FID eq {
% skip FID pair
pop pop
} {
% copy to the new font dictionary
exch newfont 3 1 roll put
} ifelse
} forall
newfont /FontName newfontname put
% insert only valid encoding vectors
encoding_vector length 256 eq {
newfont /Encoding encoding_vector put
} if
newfontname newfont definefont pop
} def
/MF_PS { % fontname newfontname -> - make a new font preserving its enc
/newfontname exch def
/fontname exch def
/fontdict fontname findfont def
/newfont fontdict maxlength dict def
fontdict {
exch
dup /FID eq {
% skip FID pair
pop pop
} {
% copy to the new font dictionary
exch newfont 3 1 roll put
} ifelse
} forall
newfont /FontName newfontname put
newfontname newfont definefont pop
} def
/SF { % fontname width height -> - set a new font
/height exch def
/width exch def
findfont
[width 0 0 height 0 0] makefont setfont
} def
/SUF { % fontname width height -> - set a new user font
/height exch def
/width exch def
/F-gs-user-font MF
/F-gs-user-font width height SF
} def
/SUF_PS { % fontname width height -> - set a new user font preserving its enc
/height exch def
/width exch def
/F-gs-user-font MF_PS
/F-gs-user-font width height SF
} def
/M {moveto} bind def
/s {show} bind def
/Box { % x y w h -> - define box path
/d_h exch def /d_w exch def /d_y exch def /d_x exch def
d_x d_y moveto
d_w 0 rlineto
0 d_h rlineto
d_w neg 0 rlineto
closepath
} def
/bgs { % x y height blskip gray str -> - show string with bg color
/str exch def
/gray exch def
/blskip exch def
/height exch def
/y exch def
/x exch def
gsave
x y blskip sub str stringwidth pop height Box
gray setgray
fill
grestore
x y M str s
} def
/bgcs { % x y height blskip red green blue str -> - show string with bg color
/str exch def
/blue exch def
/green exch def
/red exch def
/blskip exch def
/height exch def
/y exch def
/x exch def
gsave
x y blskip sub str stringwidth pop height Box
red green blue setrgbcolor
fill
grestore
x y M str s
} def
% Highlight bars.
/highlight_bars { % nlines lineheight output_y_margin gray -> -
gsave
setgray
/ymarg exch def
/lineheight exch def
/nlines exch def
% This 2 is just a magic number to sync highlight lines to text.
0 d_header_y ymarg sub 2 sub translate
/cw d_output_w cols div def
/nrows d_output_h ymarg 2 mul sub lineheight div cvi def
% for each column
0 1 cols 1 sub {
cw mul /xp exch def
% for each rows
0 1 nrows 1 sub {
/rn exch def
rn lineheight mul neg /yp exch def
rn nlines idiv 2 mod 0 eq {
% Draw highlight bar. 4 is just a magic indentation.
xp 4 add yp cw 8 sub lineheight neg Box fill
} if
} for
} for
grestore
} def
% Line highlight bar.
/line_highlight { % x y width height gray -> -
gsave
/gray exch def
Box gray setgray fill
grestore
} def
% Column separator lines.
/column_lines {
gsave
.1 setlinewidth
0 d_footer_h translate
/cw d_output_w cols div def
1 1 cols 1 sub {
cw mul 0 moveto
0 d_output_h rlineto stroke
} for
grestore
} def
% Column borders.
/column_borders {
gsave
.1 setlinewidth
0 d_footer_h moveto
0 d_output_h rlineto
d_output_w 0 rlineto
0 d_output_h neg rlineto
closepath stroke
grestore
} def
% Do the actual underlay drawing
/draw_underlay {
ul_style 0 eq {
ul_str true charpath stroke
} {
ul_str show
} ifelse
} def
% Underlay
/underlay { % - -> -
gsave
0 d_page_h translate
d_page_h neg d_page_w atan rotate
ul_gray setgray
ul_font setfont
/dw d_page_h dup mul d_page_w dup mul add sqrt def
ul_str stringwidth pop dw exch sub 2 div ul_h_ptsize -2 div moveto
draw_underlay
grestore
} def
/user_underlay { % - -> -
gsave
ul_x ul_y translate
ul_angle rotate
ul_gray setgray
ul_font setfont
0 0 ul_h_ptsize 2 div sub moveto
draw_underlay
grestore
} def
% Page prefeed
/page_prefeed { % bool -> -
statusdict /prefeed known {
statusdict exch /prefeed exch put
} {
pop
} ifelse
} def
% Wrapped line markers
/wrapped_line_mark { % x y charwith charheight type -> -
/type exch def
/h exch def
/w exch def
/y exch def
/x exch def
type 2 eq {
% Black boxes (like TeX does)
gsave
0 setlinewidth
x w 4 div add y M
0 h rlineto w 2 div 0 rlineto 0 h neg rlineto
closepath fill
grestore
} {
type 3 eq {
% Small arrows
gsave
.2 setlinewidth
x w 2 div add y h 2 div add M
w 4 div 0 rlineto
x w 4 div add y lineto stroke
x w 4 div add w 8 div add y h 4 div add M
x w 4 div add y lineto
w 4 div h 8 div rlineto stroke
grestore
} {
% do nothing
} ifelse
} ifelse
} def
% EPSF import.
/BeginEPSF {
/b4_Inc_state save def % Save state for cleanup
/dict_count countdictstack def % Count objects on dict stack
/op_count count 1 sub def % Count objects on operand stack
userdict begin
/showpage { } def
0 setgray 0 setlinecap
1 setlinewidth 0 setlinejoin
10 setmiterlimit [ ] 0 setdash newpath
/languagelevel where {
pop languagelevel
1 ne {
false setstrokeadjust false setoverprint
} if
} if
} bind def
/EndEPSF {
count op_count sub { pos } repeat % Clean up stacks
countdictstack dict_count sub { end } repeat
b4_Inc_state restore
} bind def
% Check PostScript language level.
/languagelevel where {
pop /gs_languagelevel languagelevel def
} {
/gs_languagelevel 1 def
} ifelse
%%EndResource
%%BeginResource: procset Enscript-Encoding-88591 1.6 6
/encoding_vector [
/.notdef /.notdef /.notdef /.notdef
/.notdef /.notdef /.notdef /.notdef
/.notdef /.notdef /.notdef /.notdef
/.notdef /.notdef /.notdef /.notdef
/.notdef /.notdef /.notdef /.notdef
/.notdef /.notdef /.notdef /.notdef
/.notdef /.notdef /.notdef /.notdef
/.notdef /.notdef /.notdef /.notdef
/space /exclam /quotedbl /numbersign
/dollar /percent /ampersand /quoteright
/parenleft /parenright /asterisk /plus
/comma /hyphen /period /slash
/zero /one /two /three
/four /five /six /seven
/eight /nine /colon /semicolon
/less /equal /greater /question
/at /A /B /C
/D /E /F /G
/H /I /J /K
/L /M /N /O
/P /Q /R /S
/T /U /V /W
/X /Y /Z /bracketleft
/backslash /bracketright /asciicircum /underscore
/quoteleft /a /b /c
/d /e /f /g
/h /i /j /k
/l /m /n /o
/p /q /r /s
/t /u /v /w
/x /y /z /braceleft
/bar /braceright /tilde /.notdef
/.notdef /.notdef /.notdef /.notdef
/.notdef /.notdef /.notdef /.notdef
/.notdef /.notdef /.notdef /.notdef
/.notdef /.notdef /.notdef /.notdef
/.notdef /.notdef /.notdef /.notdef
/.notdef /.notdef /.notdef /.notdef
/.notdef /.notdef /.notdef /.notdef
/.notdef /.notdef /.notdef /.notdef
/space /exclamdown /cent /sterling
/currency /yen /brokenbar /section
/dieresis /copyright /ordfeminine /guillemotleft
/logicalnot /hyphen /registered /macron
/degree /plusminus /twosuperior /threesuperior
/acute /mu /paragraph /bullet
/cedilla /onesuperior /ordmasculine /guillemotright
/onequarter /onehalf /threequarters /questiondown
/Agrave /Aacute /Acircumflex /Atilde
/Adieresis /Aring /AE /Ccedilla
/Egrave /Eacute /Ecircumflex /Edieresis
/Igrave /Iacute /Icircumflex /Idieresis
/Eth /Ntilde /Ograve /Oacute
/Ocircumflex /Otilde /Odieresis /multiply
/Oslash /Ugrave /Uacute /Ucircumflex
/Udieresis /Yacute /Thorn /germandbls
/agrave /aacute /acircumflex /atilde
/adieresis /aring /ae /ccedilla
/egrave /eacute /ecircumflex /edieresis
/igrave /iacute /icircumflex /idieresis
/eth /ntilde /ograve /oacute
/ocircumflex /otilde /odieresis /divide
/oslash /ugrave /uacute /ucircumflex
/udieresis /yacute /thorn /ydieresis
] def
%%EndResource
%%EndProlog
%%BeginSetup
%%IncludeResource: font Courier-Bold
%%IncludeResource: font Courier
/HFpt_w 10 def
/HFpt_h 10 def
/Courier-Bold /HF-gs-font MF
/HF /HF-gs-font findfont [HFpt_w 0 0 HFpt_h 0 0] makefont def
/Courier /F-gs-font MF
/F-gs-font 10 10 SF
/#copies 1 def
% Pagedevice definitions:
gs_languagelevel 1 gt {
<<
/PageSize [595 842]
>> setpagedevice
} if
/d_page_w 495 def
/d_page_h 742 def
/d_header_x 0 def
/d_header_y 742 def
/d_header_w 495 def
/d_header_h 0 def
/d_footer_x 0 def
/d_footer_y 0 def
/d_footer_w 495 def
/d_footer_h 0 def
/d_output_w 495 def
/d_output_h 742 def
/cols 1 def
%%EndSetup
%%Page: (1) 1
%%BeginPageSetup
_S
76 24 translate
/pagenum 1 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 685 M
(Network File System Version 4 C. Lever, Ed.) s
5 674 M
(Internet-Draft Oracle) s
5 663 M
(Obsoletes: 5666 \(if approved\) W. Simpson) s
5 652 M
(Intended status: Standards Track DayDreamer) s
5 641 M
(Expires: September 5, 2016 T. Talpey) s
5 630 M
( Microsoft) s
5 619 M
( March 4, 2016) s
5 586 M
( Remote Direct Memory Access Transport for Remote Procedure Call) s
5 575 M
( draft-ietf-nfsv4-rfc5666bis-04) s
5 553 M
(Abstract) s
5 531 M
( This document specifies a protocol for conveying Remote Procedure) s
5 520 M
( Call \(RPC\) messages on physical transports capable of Remote Direct) s
5 509 M
( Memory Access \(RDMA\). It requires no revision to application RPC) s
5 498 M
( protocols or the RPC protocol itself. This document obsoletes RFC) s
5 487 M
( 5666.) s
5 465 M
(Status of This Memo) s
5 443 M
( This Internet-Draft is submitted in full conformance with the) s
5 432 M
( provisions of BCP 78 and BCP 79.) s
5 410 M
( Internet-Drafts are working documents of the Internet Engineering) s
5 399 M
( Task Force \(IETF\). Note that other groups may also distribute) s
5 388 M
( working documents as Internet-Drafts. The list of current Internet-) s
5 377 M
( Drafts is at http://datatracker.ietf.org/drafts/current/.) s
5 355 M
( Internet-Drafts are draft documents valid for a maximum of six months) s
5 344 M
( and may be updated, replaced, or obsoleted by other documents at any) s
5 333 M
( time. It is inappropriate to use Internet-Drafts as reference) s
5 322 M
( material or to cite them other than as "work in progress.") s
5 300 M
( This Internet-Draft will expire on September 5, 2016.) s
5 278 M
(Copyright Notice) s
5 256 M
( Copyright \(c\) 2016 IETF Trust and the persons identified as the) s
5 245 M
( document authors. All rights reserved.) s
5 223 M
( This document is subject to BCP 78 and the IETF Trust's Legal) s
5 212 M
( Provisions Relating to IETF Documents) s
5 201 M
( \(http://trustee.ietf.org/license-info\) in effect on the date of) s
5 190 M
( publication of this document. Please review these documents) s
5 179 M
( carefully, as they describe your rights and restrictions with respect) s
5 168 M
( to this document. Code Components extracted from this document must) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 1]) s
_R
S
%%Page: (2) 2
%%BeginPageSetup
_S
76 24 translate
/pagenum 2 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( include Simplified BSD License text as described in Section 4.e of) s
5 674 M
( the Trust Legal Provisions and are provided without warranty as) s
5 663 M
( described in the Simplified BSD License.) s
5 641 M
(Table of Contents) s
5 619 M
( 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 3) s
5 608 M
( 1.1. Requirements Language . . . . . . . . . . . . . . . . . . 3) s
5 597 M
( 1.2. Remote Procedure Calls On RDMA Transports . . . . . . . . 3) s
5 586 M
( 2. Changes Since RFC 5666 . . . . . . . . . . . . . . . . . . . 4) s
5 575 M
( 2.1. Changes To The Specification . . . . . . . . . . . . . . 4) s
5 564 M
( 2.2. Changes To The XDR Definition . . . . . . . . . . . . . . 5) s
5 553 M
( 2.3. Changes To The Protocol . . . . . . . . . . . . . . . . . 5) s
5 542 M
( 3. Terminology . . . . . . . . . . . . . . . . . . . . . . . . . 6) s
5 531 M
( 3.1. Remote Procedure Calls . . . . . . . . . . . . . . . . . 6) s
5 520 M
( 3.2. Remote Direct Memory Access . . . . . . . . . . . . . . . 8) s
5 509 M
( 4. RPC-Over-RDMA Protocol Framework . . . . . . . . . . . . . . 10) s
5 498 M
( 4.1. Transfer Models . . . . . . . . . . . . . . . . . . . . . 11) s
5 487 M
( 4.2. Message Framing . . . . . . . . . . . . . . . . . . . . . 11) s
5 476 M
( 4.3. Managing Receiver Resources . . . . . . . . . . . . . . . 12) s
5 465 M
( 4.4. XDR Encoding With Chunks . . . . . . . . . . . . . . . . 14) s
5 454 M
( 4.5. Message Size . . . . . . . . . . . . . . . . . . . . . . 20) s
5 443 M
( 5. RPC-Over-RDMA In Operation . . . . . . . . . . . . . . . . . 22) s
5 432 M
( 5.1. XDR Protocol Definition . . . . . . . . . . . . . . . . . 22) s
5 421 M
( 5.2. Fixed Header Fields . . . . . . . . . . . . . . . . . . . 28) s
5 410 M
( 5.3. Chunk Lists . . . . . . . . . . . . . . . . . . . . . . . 30) s
5 399 M
( 5.4. Memory Registration . . . . . . . . . . . . . . . . . . . 32) s
5 388 M
( 5.5. Error Handling . . . . . . . . . . . . . . . . . . . . . 33) s
5 377 M
( 5.6. Protocol Elements No Longer Supported . . . . . . . . . . 36) s
5 366 M
( 5.7. XDR Examples . . . . . . . . . . . . . . . . . . . . . . 37) s
5 355 M
( 6. RPC Bind Parameters . . . . . . . . . . . . . . . . . . . . . 39) s
5 344 M
( 7. Bi-Directional RPC-Over-RDMA . . . . . . . . . . . . . . . . 40) s
5 333 M
( 7.1. RPC Direction . . . . . . . . . . . . . . . . . . . . . . 40) s
5 322 M
( 7.2. Backward Direction Flow Control . . . . . . . . . . . . . 41) s
5 311 M
( 7.3. Conventions For Backward Operation . . . . . . . . . . . 43) s
5 300 M
( 7.4. Backward Direction Upper Layer Binding . . . . . . . . . 45) s
5 289 M
( 8. Upper Layer Binding Specifications . . . . . . . . . . . . . 45) s
5 278 M
( 8.1. DDP-Eligibility . . . . . . . . . . . . . . . . . . . . . 46) s
5 267 M
( 8.2. Maximum Reply Size . . . . . . . . . . . . . . . . . . . 47) s
5 256 M
( 8.3. Additional Considerations . . . . . . . . . . . . . . . . 47) s
5 245 M
( 8.4. Upper Layer Protocol Extensions . . . . . . . . . . . . . 48) s
5 234 M
( 9. Protocol Extensibility . . . . . . . . . . . . . . . . . . . 48) s
5 223 M
( 9.1. Changes To RPC-Over-RDMA Header XDR . . . . . . . . . . . 49) s
5 212 M
( 9.2. Feature Statuses With RPC-Over-RDMA Versions . . . . . . 50) s
5 201 M
( 9.3. RPC-Over-RDMA Version Numbering . . . . . . . . . . . . . 51) s
5 190 M
( 9.4. RPC-Over-RDMA Version One Extension Practices . . . . . . 52) s
5 179 M
( 10. Security Considerations . . . . . . . . . . . . . . . . . . . 53) s
5 168 M
( 10.1. Memory Protection . . . . . . . . . . . . . . . . . . . 53) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 2]) s
_R
S
%%Page: (3) 3
%%BeginPageSetup
_S
76 24 translate
/pagenum 3 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( 10.2. RPC Message Security . . . . . . . . . . . . . . . . . . 54) s
5 674 M
( 11. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 57) s
5 663 M
( 12. Acknowledgments . . . . . . . . . . . . . . . . . . . . . . . 58) s
5 652 M
( 13. References . . . . . . . . . . . . . . . . . . . . . . . . . 58) s
5 641 M
( 13.1. Normative References . . . . . . . . . . . . . . . . . . 58) s
5 630 M
( 13.2. Informative References . . . . . . . . . . . . . . . . . 59) s
5 619 M
( Authors' Addresses . . . . . . . . . . . . . . . . . . . . . . . 61) s
5 597 M
(1. Introduction) s
5 575 M
( This document obsoletes RFC 5666. However, the protocol specified by) s
5 564 M
( this document is based on existing interoperating implementations of) s
5 553 M
( the RPC-over-RDMA Version One protocol.) s
5 531 M
( The new specification clarifies text that is subject to multiple) s
5 520 M
( interpretations, and removes support for unimplemented RPC-over-RDMA) s
5 509 M
( Version One protocol elements. It makes the role of Upper Layer) s
5 498 M
( Bindings an explicit part of the protocol specification.) s
5 476 M
( In addition, this document introduces conventions that enable bi-) s
5 465 M
( directional RPC-over-RDMA operation, enabling operation of NFSv4.1) s
5 454 M
( [RFC5661] on RDMA transports, and that enable the use of RPCSEC_GSS) s
5 443 M
( [RFC5403] on RDMA transports.) s
5 421 M
(1.1. Requirements Language) s
5 399 M
( The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT",) s
5 388 M
( "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this) s
5 377 M
( document are to be interpreted as described in [RFC2119].) s
5 355 M
(1.2. Remote Procedure Calls On RDMA Transports) s
5 333 M
( Remote Direct Memory Access \(RDMA\) [RFC5040] [RFC5041] [IB] is a) s
5 322 M
( technique for moving data efficiently between end nodes. By) s
5 311 M
( directing data into destination buffers as it is sent on a network,) s
5 300 M
( and placing it via direct memory access by hardware, the benefits of) s
5 289 M
( faster transfers and reduced host overhead are obtained.) s
5 267 M
( Open Network Computing Remote Procedure Call \(ONC RPC, or simply,) s
5 256 M
( RPC\) [RFC5531] is a remote procedure call protocol that runs over a) s
5 245 M
( variety of transports. Most RPC implementations today use UDP) s
5 234 M
( [RFC0768] or TCP [RFC0793]. On UDP, RPC messages are encapsulated) s
5 223 M
( inside datagrams, while on a TCP byte stream, RPC messages are) s
5 212 M
( delineated by a record marking protocol. An RDMA transport also) s
5 201 M
( conveys RPC messages in a specific fashion that must be fully) s
5 190 M
( described if RPC implementations are to interoperate.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 3]) s
_R
S
%%Page: (4) 4
%%BeginPageSetup
_S
76 24 translate
/pagenum 4 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( RDMA transports present semantics different from either UDP or TCP.) s
5 674 M
( They retain message delineations like UDP, but provide a reliable and) s
5 663 M
( sequenced data transfer like TCP. They also provide an offloaded) s
5 652 M
( bulk transfer service not provided by UDP or TCP. RDMA transports) s
5 641 M
( are therefore appropriately viewed as a new transport type by RPC.) s
5 619 M
( In this context, the Network File System \(NFS\) protocols as described) s
5 608 M
( in [RFC1094], [RFC1813], [RFC7530], [RFC5661], and future NFSv4 minor) s
5 597 M
( verions are obvious beneficiaries of RDMA transports. A complete) s
5 586 M
( problem statement is discussed in [RFC5532], and NFSv4-related issues) s
5 575 M
( are discussed in [RFC5661]. Many other RPC-based protocols can also) s
5 564 M
( benefit.) s
5 542 M
( Although the RDMA transport described here can provide relatively) s
5 531 M
( transparent support for any RPC application, this document also) s
5 520 M
( describes mechanisms that can optimize data transfer further, given) s
5 509 M
( more active participation by RPC applications.) s
5 487 M
(2. Changes Since RFC 5666) s
5 465 M
(2.1. Changes To The Specification) s
5 443 M
( The following alterations have been made to the RPC-over-RDMA Version) s
5 432 M
( One specification. The section numbers below refer to [RFC5666].) s
5 410 M
( o Section 2 has been expanded to introduce and explain key RPC, XDR,) s
5 399 M
( and RDMA terminology. These terms are now used consistently) s
5 388 M
( throughout the specification.) s
5 366 M
( o Section 3 has been re-organized and split into sub-sections to) s
5 355 M
( help readers locate specific requirements and definitions.) s
5 333 M
( o Sections 4 and 5 have been combined to improve the organization of) s
5 322 M
( this information.) s
5 300 M
( o The specification of the optional Connection Configuration) s
5 289 M
( Protocol has been removed from the specification.) s
5 267 M
( o A section consolidating requirements for Upper Layer Bindings has) s
5 256 M
( been added.) s
5 234 M
( o A section discussing RPC-over-RDMA protocol extensibility has been) s
5 223 M
( added.) s
5 201 M
( o A section specifying conventions for bi-directional RPC operation) s
5 190 M
( on RPC-over-RDMA Version One has been added.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 4]) s
_R
S
%%Page: (5) 5
%%BeginPageSetup
_S
76 24 translate
/pagenum 5 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( o The "Security Considerations" section has been expanded to include) s
5 674 M
( a discussion of how RPC-over-RDMA security depends on features of) s
5 663 M
( the underlying RDMA transport. A subsection specifying) s
5 652 M
( conventions for using RPCSEC_GSS with RPC-over-RDMA Version One) s
5 641 M
( has been added.) s
5 619 M
(2.2. Changes To The XDR Definition) s
5 597 M
( The XDR changes described in this section do not alter the over-the-) s
5 586 M
( wire message format described in [RFC5666]. Changes made to the XDR) s
5 575 M
( which do alter the over-the-wire message format \(i.e., to make it) s
5 564 M
( match actual interoperating implementations\) are discussed in) s
5 553 M
( Section 2.3.) s
5 531 M
( These alterations make it easier to extend the RPC-over-RDMA) s
5 520 M
( protocol. They also better organize the definition, making the) s
5 509 M
( protocol elements more consonant with actual protocol function. The) s
5 498 M
( specific changes are:) s
5 476 M
( o The XDR description has been given an extraction script using a) s
5 465 M
( sentinel sequence, matching the approach used in [RFC5662].) s
5 443 M
( o XDR data types which need to be the same in all RPC-over-RDMA) s
5 432 M
( versions have been moved to a separate section and given names) s
5 421 M
( that are not version-specific.) s
5 399 M
( o To allow extensions without modification to the existing XDR, the) s
5 388 M
( header types previously defined as members of the enum) s
5 377 M
( rpcrdma1_proc have been defined as constants, the union) s
5 366 M
( rpcrdma1_body was deleted, and RDMA_ERR_CHUNK has been renamed as) s
5 355 M
( RDMA_ERR_BADHEADER.) s
5 333 M
(2.3. Changes To The Protocol) s
5 311 M
( Although the protocol described herein interoperates with existing) s
5 300 M
( implementations of [RFC5666], the following changes have been made) s
5 289 M
( relative to the protocol described in that document:) s
5 267 M
( o Support for the Read-Read transfer model has been removed. Read-) s
5 256 M
( Read is a slower transfer model than Read-Write, thus implementers) s
5 245 M
( have chosen not to support it. Removal simplifies explanatory) s
5 234 M
( text, and support for the RDMA_DONE procedure is no longer) s
5 223 M
( necessary.) s
5 201 M
( o The specification of RDMA_MSGP in [RFC5666] and current) s
5 190 M
( implementations of it are incomplete. Even if completed, benefit) s
5 179 M
( for protocols such as NFSv4.0 [RFC7530] is doubtful. Therefore) s
5 168 M
( the RDMA_MSGP message type is no longer supported.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 5]) s
_R
S
%%Page: (6) 6
%%BeginPageSetup
_S
76 24 translate
/pagenum 6 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( o Technical errors with regard to handling RPC-over-RDMA header) s
5 674 M
( errors have been corrected.) s
5 652 M
( o Specific requirements related to handling XDR round-up and complex) s
5 641 M
( XDR data types have been added.) s
5 619 M
( o Explicit guidance is provided for sizing Write chunks, managing) s
5 608 M
( multiple chunks in the Write list, and handling unused Write) s
5 597 M
( chunks.) s
5 575 M
( o Clear guidance about Send and Receive buffer size has been added.) s
5 564 M
( This enables better decisions about when to provide and use the) s
5 553 M
( Reply chunk.) s
5 531 M
( The protocol version number has not been changed because the protocol) s
5 520 M
( specified in this document fully interoperates with implementations) s
5 509 M
( of the RPC-over-RDMA Version One protocol specified in [RFC5666].) s
5 487 M
(3. Terminology) s
5 465 M
(3.1. Remote Procedure Calls) s
5 443 M
( This section introduces key elements of the Remote Procedure Call) s
5 432 M
( [RFC5531] and External Data Representation [RFC4506] protocols, upon) s
5 421 M
( which RPC-over-RDMA Version One is constructed.) s
5 399 M
(3.1.1. Upper Layer Protocols) s
5 377 M
( Remote Procedure Calls are an abstraction used to implement the) s
5 366 M
( operations of an "Upper Layer Protocol," or ULP. The term Upper) s
5 355 M
( Layer Protocol refers to an RPC Program and Version tuple, which is a) s
5 344 M
( versioned set of procedure calls that comprise a single well-defined) s
5 333 M
( API. One example of an Upper Layer Protocol is the Network File) s
5 322 M
( System Version 4.0 [RFC7530].) s
5 300 M
(3.1.2. Requesters And Responders) s
5 278 M
( Like a local procedure call, every Remote Procedure Call \(RPC\) has a) s
5 267 M
( set of "arguments" and a set of "results". A calling context is not) s
5 256 M
( allowed to proceed until the procedure's results are available to it.) s
5 245 M
( Unlike a local procedure call, the called procedure is executed) s
5 234 M
( remotely rather than in the local application's context.) s
5 212 M
( The RPC protocol as described in [RFC5531] is fundamentally a) s
5 201 M
( message-passing protocol between one server and one or more clients.) s
5 190 M
( ONC RPC transactions are made up of two types of messages:) s
5 168 M
( CALL Message) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 6]) s
_R
S
%%Page: (7) 7
%%BeginPageSetup
_S
76 24 translate
/pagenum 7 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( A CALL message, or "Call", requests that work be done. A Call is) s
5 674 M
( designated by the value zero \(0\) in the message's msg_type field.) s
5 663 M
( An arbitrary unique value is placed in the message's xid field in) s
5 652 M
( order to match this CALL message to a corresponding REPLY message.) s
5 630 M
( REPLY Message) s
5 619 M
( A REPLY message, or "Reply", reports the results of work requested) s
5 608 M
( by a Call. A Reply is designated by the value one \(1\) in the) s
5 597 M
( message's msg_type field. The value contained in the message's) s
5 586 M
( xid field is copied from the Call whose results are being) s
5 575 M
( reported.) s
5 553 M
( The RPC client endpoint, or "requester", serializes an RPC Call's) s
5 542 M
( arguments and conveys them to a server endpoint via an RPC Call) s
5 531 M
( message. This message contains an RPC protocol header, a header) s
5 520 M
( describing the requested upper layer operation, and all arguments.) s
5 498 M
( The RPC server endpoint, or "responder", deserializes the arguments) s
5 487 M
( and processes the requested operation. It then serializes the) s
5 476 M
( operation's results into another byte stream. This byte stream is) s
5 465 M
( conveyed back to the requester via an RPC Reply message. This) s
5 454 M
( message contains an RPC protocol header, a header describing the) s
5 443 M
( upper layer reply, and all results.) s
5 421 M
( The requester deserializes the results and allows the original caller) s
5 410 M
( to proceed. At this point the RPC transaction designated by the xid) s
5 399 M
( in the Call message is complete, and the xid is retired.) s
5 377 M
(3.1.3. RPC Transports) s
5 355 M
( The role of an "RPC transport" is to mediate the exchange of RPC) s
5 344 M
( messages between requesters and responders. An RPC transport bridges) s
5 333 M
( the gap between the RPC message abstraction and the native operations) s
5 322 M
( of a particular network transport.) s
5 300 M
( RPC-over-RDMA is a connection-oriented RPC transport. When a) s
5 289 M
( connection-oriented transport is used, requesters initiate transport) s
5 278 M
( connections, while responders wait passively for incoming connection) s
5 267 M
( requests.) s
5 245 M
(3.1.4. External Data Representation) s
5 223 M
( One cannot assume that all requesters and responders internally) s
5 212 M
( represent data objects the same way. RPC uses eXternal Data) s
5 201 M
( Representation, or XDR, to translate data types and serialize) s
5 190 M
( arguments and results [RFC4506].) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 7]) s
_R
S
%%Page: (8) 8
%%BeginPageSetup
_S
76 24 translate
/pagenum 8 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( The XDR protocol encodes data independent of the endianness or size) s
5 674 M
( of host-native data types, allowing unambiguous decoding of data on) s
5 663 M
( the receiving end. RPC Programs are specified by writing an XDR) s
5 652 M
( definition of their procedures, argument data types, and result data) s
5 641 M
( types.) s
5 619 M
( XDR assumes that the number of bits in a byte \(octet\) and their order) s
5 608 M
( are the same on both endpoints and on the physical network. The) s
5 597 M
( smallest indivisible unit of XDR encoding is a group of four octets) s
5 586 M
( in little-endian order. XDR also flattens lists, arrays, and other) s
5 575 M
( complex data types so they can be conveyed as a stream of bytes.) s
5 553 M
( A serialized stream of bytes that is the result of XDR encoding is) s
5 542 M
( referred to as an "XDR stream." A sending endpoint encodes native) s
5 531 M
( data into an XDR stream and then transmits that stream to a receiver.) s
5 520 M
( A receiving endpoint decodes incoming XDR byte streams into its) s
5 509 M
( native data representation format.) s
5 487 M
(3.1.4.1. XDR Opaque Data) s
5 465 M
( Sometimes a data item must be transferred as-is, without encoding or) s
5 454 M
( decoding. Such a data item is referred to as "opaque data." XDR) s
5 443 M
( encoding places opaque data items directly into an XDR stream without) s
5 432 M
( altering their content in any way. Upper Layer Protocols or) s
5 421 M
( applications perform any needed data translation in this case.) s
5 410 M
( Examples of opaque data items include the content of files, or) s
5 399 M
( generic byte strings.) s
5 377 M
(3.1.4.2. XDR Round-up) s
5 355 M
( The number of octets in a variable-size opaque data item precedes) s
5 344 M
( that item in an XDR stream. If the size of an encoded data item is) s
5 333 M
( not a multiple of four octets, octets containing zero are added to) s
5 322 M
( the end of the item as it is encoded so that the next encoded data) s
5 311 M
( item starts on a four-octet boundary. The encoded size of the item) s
5 300 M
( is not changed by the addition of the extra octets, and the zero) s
5 289 M
( bytes are not exposed to the Upper Layer.) s
5 267 M
( This technique is referred to as "XDR round-up," and the extra octets) s
5 256 M
( are referred to as "XDR padding".) s
5 234 M
(3.2. Remote Direct Memory Access) s
5 212 M
( RPC requesters and responders can be made more efficient if large RPC) s
5 201 M
( messages are transferred by a third party such as intelligent network) s
5 190 M
( interface hardware \(data movement offload\), and placed in the) s
5 179 M
( receiver's memory so that no additional adjustment of data alignment) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 8]) s
_R
S
%%Page: (9) 9
%%BeginPageSetup
_S
76 24 translate
/pagenum 9 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( has to be made \(direct data placement\). Remote Direct Memory Access) s
5 674 M
( enables both optimizations.) s
5 652 M
(3.2.1. Direct Data Placement) s
5 630 M
( Typically, RPC implementations copy the contents of RPC messages into) s
5 619 M
( a buffer before being sent. An efficient RPC implementation sends) s
5 608 M
( bulk data without copying it into a separate send buffer first.) s
5 586 M
( However, socket-based RPC implementations are often unable to receive) s
5 575 M
( data directly into its final place in memory. Receivers often need) s
5 564 M
( to copy incoming data to finish an RPC operation; sometimes, only to) s
5 553 M
( adjust data alignment.) s
5 531 M
( In this document, "RDMA" refers to the physical mechanism an RDMA) s
5 520 M
( transport utilizes when moving data. Although this may not be) s
5 509 M
( efficient, before an RDMA transfer a sender may copy data into an) s
5 498 M
( intermediate buffer before an RDMA transfer. After an RDMA transfer,) s
5 487 M
( a receiver may copy that data again to its final destination.) s
5 465 M
( This document uses the term "direct data placement" \(or DDP\) to refer) s
5 454 M
( specifically to an optimized data transfer where it is unnecessary) s
5 443 M
( for a receiving host's CPU to copy transferred data to another) s
5 432 M
( location after it has been received. Not all RDMA-based data) s
5 421 M
( transfer qualifies as Direct Data Placement, and DDP can be achieved) s
5 410 M
( using non-RDMA mechanisms.) s
5 388 M
(3.2.2. RDMA Transport Requirements) s
5 366 M
( The RPC-over-RDMA Version One protocol assumes the physical transport) s
5 355 M
( provides the following abstract operations. A more complete) s
5 344 M
( discussion of these operations is found in [RFC5040].) s
5 322 M
( Registered Memory) s
5 311 M
( Registered memory is a segment of memory that is assigned a) s
5 300 M
( steering tag that temporarily permits access by the RDMA provider) s
5 289 M
( to perform data transfer operations. The RPC-over-RDMA Version) s
5 278 M
( One protocol assumes that each segment of registered memory MUST) s
5 267 M
( be identified with a steering tag of no more than 32 bits and) s
5 256 M
( memory addresses of up to 64 bits in length.) s
5 234 M
( RDMA Send) s
5 223 M
( The RDMA provider supports an RDMA Send operation, with completion) s
5 212 M
( signaled on the receiving peer after data has been placed in a) s
5 201 M
( pre-posted memory segment. Sends complete at the receiver in the) s
5 190 M
( order they were issued at the sender. The amount of data) s
5 179 M
( transferred by an RDMA Send operation is limited by the size of) s
5 168 M
( the remote pre-posted memory segment.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 9]) s
_R
S
%%Page: (10) 10
%%BeginPageSetup
_S
76 24 translate
/pagenum 10 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( RDMA Receive) s
5 674 M
( The RDMA provider supports an RDMA Receive operation to receive) s
5 663 M
( data conveyed by incoming RDMA Send operations. To reduce the) s
5 652 M
( amount of memory that must remain pinned awaiting incoming Sends,) s
5 641 M
( the amount of pre-posted memory is limited. Flow-control to) s
5 630 M
( prevent overrunning receiver resources is provided by the RDMA) s
5 619 M
( consumer \(in this case, the RPC-over-RDMA Version One protocol\).) s
5 597 M
( RDMA Write) s
5 586 M
( The RDMA provider supports an RDMA Write operation to directly) s
5 575 M
( place data in remote memory. The local host initiates an RDMA) s
5 564 M
( Write, and completion is signaled there. No completion is) s
5 553 M
( signaled on the remote. The local host provides a steering tag,) s
5 542 M
( memory address, and length of the remote's memory segment.) s
5 520 M
( RDMA Writes are not necessarily ordered with respect to one) s
5 509 M
( another, but are ordered with respect to RDMA Sends. A subsequent) s
5 498 M
( RDMA Send completion obtained at the write initiator guarantees) s
5 487 M
( that prior RDMA Write data has been successfully placed in the) s
5 476 M
( remote peer's memory.) s
5 454 M
( RDMA Read) s
5 443 M
( The RDMA provider supports an RDMA Read operation to directly) s
5 432 M
( place peer source data in the read initiator's memory. The local) s
5 421 M
( host initiates an RDMA Read, and completion is signaled there; no) s
5 410 M
( completion is signaled on the remote. The local host provides) s
5 399 M
( steering tags, memory addresses, and a length for the remote) s
5 388 M
( source and local destination memory segments.) s
5 366 M
( The remote peer receives no notification of RDMA Read completion.) s
5 355 M
( The local host signals completion as part of a subsequent RDMA) s
5 344 M
( Send message so that the remote peer can release steering tags and) s
5 333 M
( subsequently free associated source memory segments.) s
5 311 M
( The RPC-over-RDMA Version One protocol is designed to be carried over) s
5 300 M
( RDMA transports that support the above abstract operations. This) s
5 289 M
( protocol conveys to the RPC peer information sufficient for that RPC) s
5 278 M
( peer to direct an RDMA layer to perform transfers containing RPC data) s
5 267 M
( and to communicate their result\(s\). For example, it is readily) s
5 256 M
( carried over RDMA transports such as Internet Wide Area RDMA Protocol) s
5 245 M
( \(iWARP\) [RFC5040] [RFC5041].) s
5 223 M
(4. RPC-Over-RDMA Protocol Framework) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 10]) s
_R
S
%%Page: (11) 11
%%BeginPageSetup
_S
76 24 translate
/pagenum 11 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
(4.1. Transfer Models) s
5 663 M
( A "transfer model" designates which endpoint is responsible for) s
5 652 M
( performing RDMA Read and Write operations. To enable these) s
5 641 M
( operations, the peer endpoint first exposes segments of its memory to) s
5 630 M
( the endpoint performing the RDMA Read and Write operations.) s
5 608 M
( Read-Read) s
5 597 M
( Requesters expose their memory to the responder, and the responder) s
5 586 M
( exposes its memory to requesters. The responder employs RDMA Read) s
5 575 M
( operations to pull RPC arguments or whole RPC calls from the) s
5 564 M
( requester. Requesters employ RDMA Read operations to pull RPC) s
5 553 M
( results or whole RPC relies from the responder.) s
5 531 M
( Write-Write) s
5 520 M
( Requesters expose their memory to the responder, and the responder) s
5 509 M
( exposes its memory to requesters. Requesters employ RDMA Write) s
5 498 M
( operations to push RPC arguments or whole RPC calls to the) s
5 487 M
( responder. The responder employs RDMA Write operations to push) s
5 476 M
( RPC results or whole RPC relies to the requester.) s
5 454 M
( Read-Write) s
5 443 M
( Requesters expose their memory to the responder, but the responder) s
5 432 M
( does not expose its memory. The responder employs RDMA Read) s
5 421 M
( operations to pull RPC arguments or whole RPC calls from the) s
5 410 M
( requester. The responder employs RDMA Write operations to push) s
5 399 M
( RPC results or whole RPC relies to the requester.) s
5 377 M
( Write-Read) s
5 366 M
( The responder exposes its memory to requesters, but requesters do) s
5 355 M
( not expose their memory. Requesters employ RDMA Write operations) s
5 344 M
( to push RPC arguments or whole RPC calls to the responder.) s
5 333 M
( Requesters employ RDMA Read operations to pull RPC results or) s
5 322 M
( whole RPC relies from the responder.) s
5 300 M
( [RFC5666] specifies the use of both the Read-Read and the Read-Write) s
5 289 M
( Transfer Model. All current RPC-over-RDMA Version One) s
5 278 M
( implementations use only the Read-Write Transfer Model. Therefore) s
5 267 M
( the use of the Read-Read Transfer Model by RPC-over-RDMA Version One) s
5 256 M
( implementations is no longer supported. Other Transfer Models may be) s
5 245 M
( used by a future version of RPC-over-RDMA.) s
5 223 M
(4.2. Message Framing) s
5 201 M
( On an RPC-over-RDMA transport, each RPC message is encapsulated by an) s
5 190 M
( RPC-over-RDMA message. An RPC-over-RDMA message consists of two XDR) s
5 179 M
( streams.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 11]) s
_R
S
%%Page: (12) 12
%%BeginPageSetup
_S
76 24 translate
/pagenum 12 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( RPC Payload Stream) s
5 674 M
( The "Payload stream" contains the encapsulated RPC message being) s
5 663 M
( transferred by this RPC-over-RDMA message. This stream always) s
5 652 M
( begins with the XID field of the encapsulated RPC message.) s
5 630 M
( Transport Header Stream) s
5 619 M
( The "Transport stream" contains a header that describes and) s
5 608 M
( controls the transfer of the Payload stream in this RPC-over-RDMA) s
5 597 M
( message. This header is analogous to the record marking used for) s
5 586 M
( RPC over TCP but is more extensive, since RDMA transports support) s
5 575 M
( several modes of data transfer.) s
5 553 M
( In its simplest form, an RPC-over-RDMA message consists of a) s
5 542 M
( Transport stream followed immediately by a Payload stream conveyed) s
5 531 M
( together in a single RDMA Send. To transmit large RPC messages, a) s
5 520 M
( combination of one RDMA Send operation and one or more RDMA Read or) s
5 509 M
( Write operations is employed.) s
5 487 M
( RPC-over-RDMA framing replaces all other RPC framing \(such as TCP) s
5 476 M
( record marking\) when used atop an RPC-over-RDMA association, even) s
5 465 M
( when the underlying RDMA protocol may itself be layered atop a) s
5 454 M
( transport with a defined RPC framing \(such as TCP\).) s
5 432 M
( It is however possible for RPC-over-RDMA to be dynamically enabled in) s
5 421 M
( the course of negotiating the use of RDMA via an Upper Layer Protocol) s
5 410 M
( exchange. Because RPC framing delimits an entire RPC request or) s
5 399 M
( reply, the resulting shift in framing must occur between distinct RPC) s
5 388 M
( messages, and in concert with the underlying transport.) s
5 366 M
(4.3. Managing Receiver Resources) s
5 344 M
( It is critical to provide RDMA Send flow control for an RDMA) s
5 333 M
( connection. If no pre-posted receive buffer is large enough to) s
5 322 M
( accept an incoming RDMA Send, the RDMA Send operation fails. If a) s
5 311 M
( pre-posted receive buffer is not available to accept an incoming RDMA) s
5 300 M
( Send, the RDMA Send operation can fail. Repeated occurrences of such) s
5 289 M
( errors can be fatal to the connection. This is a departure from) s
5 278 M
( conventional TCP/IP networking where buffers are allocated) s
5 267 M
( dynamically as part of receiving messages.) s
5 245 M
( The longevity of an RDMA connection requires that sending endpoints) s
5 234 M
( respect the resource limits of peer receivers. To ensure messages) s
5 223 M
( can be sent and received reliably, there are two operational) s
5 212 M
( parameters for each connection.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 12]) s
_R
S
%%Page: (13) 13
%%BeginPageSetup
_S
76 24 translate
/pagenum 13 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
(4.3.1. Credit Limit) s
5 663 M
( The number of pre-posted RDMA Receive operations is sometimes) s
5 652 M
( referred to as a peer's "credit limit." Flow control for RDMA Send) s
5 641 M
( operations directed to the responder is implemented as a simple) s
5 630 M
( request/grant protocol in the RPC-over-RDMA header associated with) s
5 619 M
( each RPC message. Section 5.2.3 has further detail.) s
5 597 M
( o The RPC-over-RDMA header for RPC Call messages contains a) s
5 586 M
( requested credit value for the responder. This is the maximum) s
5 575 M
( number of RPC replies the requester can handle at once,) s
5 564 M
( independent of how many RPCs are in flight at that moment. The) s
5 553 M
( requester MAY dynamically adjust the requested credit value to) s
5 542 M
( match its expected needs.) s
5 520 M
( o The RPC-over-RDMA header for RPC Reply messages provides the) s
5 509 M
( granted result. This is the maximum number of RPC calls the) s
5 498 M
( responder can handle at once, without regard to how many RPCs are) s
5 487 M
( in flight at that moment. The granted value MUST NOT be zero,) s
5 476 M
( since such a value would result in deadlock. The responder MAY) s
5 465 M
( dynamically adjust the granted credit value to match its needs or) s
5 454 M
( policies \(e.g. to accommodate the available resources in a shared) s
5 443 M
( receive queue\).) s
5 421 M
( The requester MUST NOT send unacknowledged requests in excess of this) s
5 410 M
( granted responder credit limit. If the limit is exceeded, the RDMA) s
5 399 M
( layer may signal an error, possibly terminating the connection. If) s
5 388 M
( an RDMA layer error does not occur, the responder MAY handle excess) s
5 377 M
( requests or return an RPC layer error to the requester.) s
5 355 M
( While RPC calls complete in any order, the current flow control limit) s
5 344 M
( at the responder is known to the requester from the Send ordering) s
5 333 M
( properties. It is always the lower of the requested and granted) s
5 322 M
( credit values, minus the number of requests in flight. Advertised) s
5 311 M
( credit values are not altered when individual RPCs are started or) s
5 300 M
( completed.) s
5 278 M
( On occasion a requester or responder may need to adjust the amount of) s
5 267 M
( resources available to a connection. When this happens, the) s
5 256 M
( responder needs to ensure that a credit increase is effected \(i.e.) s
5 245 M
( RDMA Receives are posted\) before the next reply is sent.) s
5 223 M
( Certain RDMA implementations may impose additional flow control) s
5 212 M
( restrictions, such as limits on RDMA Read operations in progress at) s
5 201 M
( the responder. Accommodation of such restrictions is considered the) s
5 190 M
( responsibility of each RPC-over-RDMA Version One implementation.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 13]) s
_R
S
%%Page: (14) 14
%%BeginPageSetup
_S
76 24 translate
/pagenum 14 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
(4.3.2. Inline Threshold) s
5 663 M
( A receiver's "inline threshold" value is the largest message size \(in) s
5 652 M
( octets\) that the receiver can accept via an RDMA Receive operation.) s
5 641 M
( Each connection has two inline threshold values, one for each peer) s
5 630 M
( receiver.) s
5 608 M
( Unlike credit limits, inline threshold values are not advertised to) s
5 597 M
( peers via the RPC-over-RDMA Version One protocol, and there is no) s
5 586 M
( provision for the inline threshold value to change during the) s
5 575 M
( lifetime of an RPC-over-RDMA Version One connection.) s
5 553 M
(4.3.3. Initial Connection State) s
5 531 M
( When a connection is first established, peers might not know how many) s
5 520 M
( receive buffers the other has, nor how large these buffers are.) s
5 498 M
( As a basis for an initial exchange of RPC requests, each RPC-over-) s
5 487 M
( RDMA Version One connection provides the ability to exchange at least) s
5 476 M
( one RPC message at a time that is 1024 bytes in size. A responder) s
5 465 M
( MAY exceed this basic level of configuration, but a requester MUST) s
5 454 M
( NOT assume more than one credit is available, and MUST receive a) s
5 443 M
( valid reply from the responder carrying the actual number of) s
5 432 M
( available credits, prior to sending its next request.) s
5 410 M
( Receiver implementations MUST support an inline threshold of 1024) s
5 399 M
( bytes, but MAY support larger inline thresholds values. A mechanism) s
5 388 M
( for discovering a peer's inline threshold value before a connection) s
5 377 M
( is established may be used to optimize the use of RDMA Send) s
5 366 M
( operations. In the absense of such a mechanism, senders MUST assume) s
5 355 M
( a receiver's inline threshold is 1024 bytes.) s
5 333 M
(4.4. XDR Encoding With Chunks) s
5 311 M
( When a direct data placement capability is available, during XDR) s
5 300 M
( encoding it can be determined that an XDR data item is large enough) s
5 289 M
( that it might be more efficient if the transport placed the content) s
5 278 M
( of the data item directly in the receiver's memory.) s
5 256 M
(4.4.1. Reducing An XDR Stream) s
5 234 M
( RPC-over-RDMA Version One provides a mechanism for moving part of an) s
5 223 M
( RPC message via a data transfer separate from an RDMA Send/Receive.) s
5 212 M
( The sender removes one or more XDR data items from the Payload) s
5 201 M
( stream. They are conveyed via one or more RDMA Read or Write) s
5 190 M
( operations. The receiver inserts the data items into the Payload) s
5 179 M
( stream before passing it to the Upper Layer.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 14]) s
_R
S
%%Page: (15) 15
%%BeginPageSetup
_S
76 24 translate
/pagenum 15 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( A contiguous piece of a Payload stream that is split out and moved) s
5 674 M
( via separate RDMA operations is known as a "chunk." A Payload stream) s
5 663 M
( after chunks have been removed is referred to as a "reduced" Payload) s
5 652 M
( stream.) s
5 630 M
(4.4.2. DDP-Eligibility) s
5 608 M
( Only an XDR data item that might benefit from Direct Data Placement) s
5 597 M
( may be reduced. The eligibility of particular XDR data items to be) s
5 586 M
( reduced is not specified by this document.) s
5 564 M
( To maintain interoperability on an RPC-over-RDMA transport, a) s
5 553 M
( determination must be made of which XDR data items in each Upper) s
5 542 M
( Layer Protocol are allowed to use Direct Data Placement. Therefore) s
5 531 M
( an additional specification is needed that describes how an Upper) s
5 520 M
( Layer Protocol enables Direct Data Placement. The set of) s
5 509 M
( requirements for an Upper Layer Protocol to use an RPC-over-RDMA) s
5 498 M
( transport is known as an "Upper Layer Binding specification," or ULB.) s
5 476 M
( An Upper Layer Binding specification states which specific individual) s
5 465 M
( XDR data items in an Upper Layer Protocol MAY be transferred via) s
5 454 M
( Direct Data Placement. This document will refer to XDR data items) s
5 443 M
( that are permitted to be reduced as "DDP-eligible". All other XDR) s
5 432 M
( data items MUST NOT be reduced. RPC-over-RDMA Version One uses RDMA) s
5 421 M
( Read and Write operations to transfer DDP-eligible data that has been) s
5 410 M
( reduced.) s
5 388 M
( Detailed requirements for Upper Layer Bindings are discussed in full) s
5 377 M
( in Section 8.) s
5 355 M
(4.4.3. RDMA Segments) s
5 333 M
( When encoding a Payload stream that contains a DDP-eligible data) s
5 322 M
( item, a sender may choose to reduce that data item. It does not) s
5 311 M
( place the item into the Payload stream. Instead, the sender records) s
5 300 M
( in the RPC-over-RDMA header the actual address and size of the memory) s
5 289 M
( region containing that data item.) s
5 267 M
( The requester provides location information for DDP-eligible data) s
5 256 M
( items in both RPC Calls and Replies. The responder uses this) s
5 245 M
( information to initiate RDMA Read and Write operations to retrieve or) s
5 234 M
( update the specified region of the requester's memory.) s
5 212 M
( An "RDMA segment", or a "plain segment", is an RPC-over-RDMA header) s
5 201 M
( data object that contains the precise co-ordinates of a contiguous) s
5 190 M
( memory region that is to be conveyed via one or more RDMA Read or) s
5 179 M
( RDMA Write operations. The following fields are contained in each) s
5 168 M
( segment.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 15]) s
_R
S
%%Page: (16) 16
%%BeginPageSetup
_S
76 24 translate
/pagenum 16 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+) s
5 674 M
( | Handle |) s
5 663 M
( +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+) s
5 652 M
( | Length |) s
5 641 M
( +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+) s
5 630 M
( | |) s
5 619 M
( + Offset +) s
5 608 M
( | |) s
5 597 M
( +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+) s
5 564 M
( Handle) s
5 553 M
( Steering tag \(STag\) or handle obtained when the segment's memory) s
5 542 M
( is registered for RDMA. Also known as an R_key, this value is) s
5 531 M
( generated by registering this memory with the RDMA provider.) s
5 509 M
( Length) s
5 498 M
( The length of the memory segment, in octets.) s
5 476 M
( Offset) s
5 465 M
( The offset or beginning memory address of the segment.) s
5 443 M
( See [RFC5040] for further discussion of the meaning of these fields.) s
5 421 M
(4.4.4. Chunks) s
5 399 M
( In RPC-over-RDMA Version One, a "chunk" refers to a portion of the) s
5 388 M
( Payload stream that is moved via RDMA Read or Write operations.) s
5 377 M
( Chunk data is removed from the sender's Payload stream, transferred) s
5 366 M
( by separate RDMA operations, and then re-inserted into the receiver's) s
5 355 M
( Payload stream.) s
5 333 M
( Each chunk consists of one or more RDMA segments. Each segment) s
5 322 M
( represents a single contiguous piece of that chunk. Segments MAY) s
5 311 M
( divide a chunk on any boundary that is convenient to the requester.) s
5 289 M
( Except in special cases, a chunk contains exactly one XDR data item.) s
5 278 M
( This makes it straightforward to remove chunks from an XDR stream) s
5 267 M
( without affecting XDR alignment. Not every RPC-over-RDMA message has) s
5 256 M
( chunks associated with it.) s
5 234 M
(4.4.4.1. Counted Arrays) s
5 212 M
( If a chunk contains a counted array data type, the count of array) s
5 201 M
( elements MUST remain in the Payload stream, while the array elements) s
5 190 M
( MUST be moved to the chunk. For example, when encoding an opaque) s
5 179 M
( byte array as a chunk, the count of bytes stays in the Payload) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 16]) s
_R
S
%%Page: (17) 17
%%BeginPageSetup
_S
76 24 translate
/pagenum 17 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( stream, while the bytes in the array are removed from the Payload) s
5 674 M
( stream and transferred within the chunk.) s
5 652 M
( Any byte count left in the Payload stream MUST match the sum of the) s
5 641 M
( lengths of the segments making up the chunk. If they do not agree,) s
5 630 M
( an RPC protocol encoding error results.) s
5 608 M
( Individual array elements appear in a chunk in their entirety. For) s
5 597 M
( example, when encoding an array of arrays as a chunk, the count of) s
5 586 M
( items in the enclosing array stays in the Payload stream, but each) s
5 575 M
( enclosed array, including its item count, is transferred as part of) s
5 564 M
( the chunk.) s
5 542 M
(4.4.4.2. Optional-data) s
5 520 M
( If a chunk contains an optional-data data type, the "is present") s
5 509 M
( field MUST remain in the Payload stream, while the data, if present,) s
5 498 M
( MUST be moved to the chunk.) s
5 476 M
(4.4.4.3. XDR Unions) s
5 454 M
( A union data type should never be made DDP-eligible, but one or more) s
5 443 M
( of its arms may be DDP-eligible.) s
5 421 M
(4.4.5. Read Chunks) s
5 399 M
( A "Read chunk" represents an XDR data item that is to be pulled from) s
5 388 M
( the requester to the responder using RDMA Read operations.) s
5 366 M
( A Read chunk is a list of one or more RDMA segments. Each RDMA) s
5 355 M
( segment in a Read chunk is a plain segment which has an additional) s
5 344 M
( Position field.) s
5 311 M
( +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+) s
5 300 M
( | Position |) s
5 289 M
( +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+) s
5 278 M
( | Handle |) s
5 267 M
( +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+) s
5 256 M
( | Length |) s
5 245 M
( +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+) s
5 234 M
( | |) s
5 223 M
( + Offset +) s
5 212 M
( | |) s
5 201 M
( +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+) s
5 168 M
( Position) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 17]) s
_R
S
%%Page: (18) 18
%%BeginPageSetup
_S
76 24 translate
/pagenum 18 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( The byte offset in the Payload stream where the receiver re-) s
5 674 M
( inserts the data item conveyed in a chunk. The Position value) s
5 663 M
( MUST be computed from the beginning of the Payload stream, which) s
5 652 M
( begins at Position zero. All RDMA segments belonging to the same) s
5 641 M
( Read chunk have the same value in their Position field.) s
5 619 M
( While constructing an RPC-over-RDMA Call message, a requester) s
5 608 M
( registers memory segments containing data in Read chunks. It) s
5 597 M
( advertises these chunks in the RPC-over-RDMA header of the RPC Call.) s
5 575 M
( After receiving an RPC Call sent via an RDMA Send operation, a) s
5 564 M
( responder transfers the chunk data from the requester using RDMA Read) s
5 553 M
( operations. The responder reconstructs the transferred chunk data by) s
5 542 M
( concatenating the contents of each segment, in list order, into the) s
5 531 M
( received Payload stream at the Position value recorded in the) s
5 520 M
( segment.) s
5 498 M
( Put another way, a receiver inserts the first segment in a Read chunk) s
5 487 M
( into the Payload stream at the byte offset indicated by its Position) s
5 476 M
( field. Segments whose Position field value match this offset are) s
5 465 M
( concatenated afterwards, until there are no more segments at that) s
5 454 M
( Position value. The next XDR data item in the Payload stream) s
5 443 M
( follows.) s
5 421 M
(4.4.5.1. Read Chunk Round-up) s
5 399 M
( XDR requires each encoded data item to start on four-byte alignment.) s
5 388 M
( When an odd-length data item is encoded, its length is encoded) s
5 377 M
( literally, while the data is padded so the next data item in the XDR) s
5 366 M
( stream can start on a four-byte boundary. Receivers ignore the) s
5 355 M
( content of the pad bytes.) s
5 333 M
( After an XDR data item has been reduced, all data items remaining in) s
5 322 M
( the Payload stream must continue to adhere to these padding) s
5 311 M
( requirements. Thus when an XDR data item is moved from the Payload) s
5 300 M
( stream into a Read chunk, the requester MUST remove XDR padding for) s
5 289 M
( that data item from the Payload stream as well.) s
5 267 M
( The length of a Read chunk is the sum of the lengths of the read) s
5 256 M
( segments that comprise it. If this sum is not a multiple of four,) s
5 245 M
( the requester MAY choose to send a Read chunk without any XDR) s
5 234 M
( padding. If the requester provides no actual round-up in a Read) s
5 223 M
( chunk, the responder MUST be prepared to provide appropriate round-up) s
5 212 M
( in the reconstructed call XDR stream) s
5 190 M
( The Position field in a read segment indicates where the containing) s
5 179 M
( Read chunk starts in the Payload stream. The value in this field) s
5 168 M
( MUST be a multiple of four. Moreover, all segments in the same Read) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 18]) s
_R
S
%%Page: (19) 19
%%BeginPageSetup
_S
76 24 translate
/pagenum 19 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( chunk share the same Position value, even if one or more of the) s
5 674 M
( segments have a non-four-byte aligned length.) s
5 652 M
(4.4.5.2. Decoding Read Chunks) s
5 630 M
( While decoding a received Payload stream, whenever the XDR offset in) s
5 619 M
( the Payload stream matches that of a Read chunk, the transport) s
5 608 M
( initiates an RDMA Read to pull the chunk's data content into) s
5 597 M
( registered memory on the responder.) s
5 575 M
( The responder acknowledges its completion of use of Read chunk source) s
5 564 M
( buffers when it sends an RPC Reply to the requester. The requester) s
5 553 M
( may then release Read chunks advertised in the request.) s
5 531 M
(4.4.6. Write Chunks) s
5 509 M
( A "Write chunk" represents an XDR data item that is to be pushed from) s
5 498 M
( a responder to a requester using RDMA Write operations.) s
5 476 M
( A Write chunk is an array of one or more plain RDMA segments. Write) s
5 465 M
( chunks are provided by a requester long before the responder has) s
5 454 M
( prepared the reply Payload stream. Therefore RDMA segments in a) s
5 443 M
( Write chunk do not have a Position field.) s
5 421 M
( While constructing an RPC Call message, a requester also prepares) s
5 410 M
( memory regions to catch DDP-eligible reply data items. A requester) s
5 399 M
( does not know the actual length of the result data item to be) s
5 388 M
( returned, thus it MUST register a Write chunk long enough to) s
5 377 M
( accommodate the maximum possible size of the returned data item.) s
5 355 M
( A responder copies the requester-provided Write chunk segments into) s
5 344 M
( the RPC-over-RDMA header that it returns with the reply. The) s
5 333 M
( responder MUST NOT change the number of segments in the Write chunk.) s
5 311 M
( The responder fills the segments in array order until the data item) s
5 300 M
( has been completely written. The responder updates the segment) s
5 289 M
( length fields to reflect the actual amount of data that is being) s
5 278 M
( returned in each segment. If a Write chunk segment is not filled by) s
5 267 M
( the responder, the updated length of the segment SHOULD be zero.) s
5 245 M
( The responder then sends the RPC Reply via an RDMA Send operation.) s
5 234 M
( After receiving the RPC Reply, the requester reconstructs the) s
5 223 M
( transferred data by concatenating the contents of each segment, in) s
5 212 M
( array order, into RPC Reply XDR stream.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 19]) s
_R
S
%%Page: (20) 20
%%BeginPageSetup
_S
76 24 translate
/pagenum 20 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
(4.4.6.1. Write Chunk Round-up) s
5 663 M
( XDR requires each encoded data item to start on four-byte alignment.) s
5 652 M
( When an odd-length data item is encoded, its length is encoded) s
5 641 M
( literally, while the data is padded so the next data item in the XDR) s
5 630 M
( stream can start on a four-byte boundary. Receivers ignore the) s
5 619 M
( content of the pad bytes.) s
5 597 M
( After a data item is reduced, data items remaining in the Payload) s
5 586 M
( stream must continue to adhere to these padding requirements. Thus) s
5 575 M
( when an XDR data item is moved from a reply Payload stream into a) s
5 564 M
( Write chunk, the responder MUST remove XDR padding for that data item) s
5 553 M
( from the reply Payload stream as well.) s
5 531 M
( A requester SHOULD NOT provide extra length in a Write chunk to) s
5 520 M
( accommodate XDR pad bytes. A responder MUST NOT write XDR pad bytes) s
5 509 M
( for a Write chunk.) s
5 487 M
(4.4.6.2. Unused Write Chunks) s
5 465 M
( There are occasions when a requester provides a Write chunk but the) s
5 454 M
( responder does not use it.) s
5 432 M
( For example, an Upper Layer Protocol may define a union result where) s
5 421 M
( some arms of the union contain a DDP-eligible data item while other) s
5 410 M
( arms do not. The requester is required to provide a Write chunk in) s
5 399 M
( this case, but if the responder returns a result that uses an arm of) s
5 388 M
( the union that has no DDP-eligible data item, the Write chunk remains) s
5 377 M
( unused.) s
5 355 M
( When forming an RPC-over-RDMA Reply message with an unused Write) s
5 344 M
( chunk, the responder MUST set the length of all segments in the chunk) s
5 333 M
( to zero.) s
5 311 M
( Unused write chunks, or unused bytes in write chunk segments, are not) s
5 300 M
( returned as results. Their memory is returned to the Upper Layer as) s
5 289 M
( part of RPC completion. However, the RPC layer MUST NOT assume that) s
5 278 M
( the buffers have not been modified.) s
5 256 M
(4.5. Message Size) s
5 234 M
( A receiver of RDMA Send operations is required by RDMA to have) s
5 223 M
( previously posted one or more adequately sized buffers. Memory) s
5 212 M
( savings can be achieved on both requesters and responders by leaving) s
5 201 M
( the inline threshold small. However, not all RPC messages are small.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 20]) s
_R
S
%%Page: (21) 21
%%BeginPageSetup
_S
76 24 translate
/pagenum 21 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
(4.5.1. Short Messages) s
5 663 M
( RPC messages are frequently smaller than typical inline thresholds.) s
5 652 M
( For example, the NFS version 3 GETATTR request is only 56 bytes: 20) s
5 641 M
( bytes of RPC header, plus a 32-byte file handle argument and 4 bytes) s
5 630 M
( for its length. The reply to this common request is about 100 bytes.) s
5 608 M
( Since all RPC messages conveyed via RPC-over-RDMA require an RDMA) s
5 597 M
( Send operation, the most efficient way to send an RPC message that is) s
5 586 M
( smaller than the receiver's inline threshold is to append the Payload) s
5 575 M
( stream directly to the Transport stream. An RPC-over-RDMA header) s
5 564 M
( with a small RPC Call or Reply message immediately following is) s
5 553 M
( transferred using a single RDMA Send operation. No RDMA Read or) s
5 542 M
( Write operations are needed.) s
5 520 M
(4.5.2. Chunked Messages) s
5 498 M
( If DDP-eligible data items are present in a Payload stream, a sender) s
5 487 M
( MAY reduce the Payload stream to enable the use of RDMA Read or Write) s
5 476 M
( operations to move the reduced data items. The Transport stream with) s
5 465 M
( the reduced Payload stream immediately following is transferred using) s
5 454 M
( a single RDMA Send operation.) s
5 432 M
( After receiving the Transport and Payload streams of a Chunked RPC-) s
5 421 M
( over-RDMA Call message, the responder uses RDMA Read operations to) s
5 410 M
( move reduced data items in Read chunks. Before sending the Transport) s
5 399 M
( and Payload streams of a Chunked RPC-over-RDMA Reply message, the) s
5 388 M
( responder uses RDMA Write operations to move reduced data items in) s
5 377 M
( Write and Reply chunks.) s
5 355 M
(4.5.3. Long Messages) s
5 333 M
( When a Payload stream is larger than the receiver's inline threshold,) s
5 322 M
( the Payload stream is reduced by removing DDP-eligible data items and) s
5 311 M
( placing them in chunks to be moved separately. If there are no DDP-) s
5 300 M
( eligible data items in the Payload stream, or the Payload stream is) s
5 289 M
( still too large after it has been reduced, the RDMA transport MUST) s
5 278 M
( use RDMA Read or Write operations to convey the Payload stream) s
5 267 M
( itself. This mechanism is referred to as a "Long Message.") s
5 245 M
( To transmit a Long Message, the sender conveys only the Transport) s
5 234 M
( stream with an RDMA Send operation. The Payload stream is not) s
5 223 M
( included in the Send buffer in this instance. Instead, the requester) s
5 212 M
( provides chunks that the responder uses to move the Payload stream.) s
5 190 M
( Long RPC Call) s
5 179 M
( To send a Long RPC-over-RDMA Call message, the requester provides) s
5 168 M
( a special Read chunk that contains the RPC Call's Payload stream.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 21]) s
_R
S
%%Page: (22) 22
%%BeginPageSetup
_S
76 24 translate
/pagenum 22 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( Every segment in this Read chunk MUST contain zero in its Position) s
5 674 M
( field. Thus this chunk is known as a "Position Zero Read chunk.") s
5 652 M
( Long RPC Reply) s
5 641 M
( To send a Long RPC-over-RDMA Reply message, the requester provides) s
5 630 M
( a single special Write chunk in advance, known as the "Reply) s
5 619 M
( chunk", that will contain the RPC Reply's Payload stream. The) s
5 608 M
( requester sizes the Reply chunk to accommodate the maximum) s
5 597 M
( expected reply size for that Upper Layer operation.) s
5 575 M
( Though the purpose of a Long Message is to handle large RPC messages,) s
5 564 M
( requesters MAY use a Long Message at any time to convey an RPC Call.) s
5 542 M
( A responder chooses which form of reply to use based on the chunks) s
5 531 M
( provided by the requester. If Write chunks were provided and the) s
5 520 M
( responder has a DDP-eligible result, it first reduces the reply) s
5 509 M
( Payload stream. If a Reply chunk was provided and the reduced) s
5 498 M
( Payload is larger than the requester's inline threshold, the) s
5 487 M
( responder MUST use the provided Reply chunk for the reply.) s
5 465 M
( Because these special chunks contain a whole RPC message, any XDR) s
5 454 M
( data item MAY appear in one of these special chunks without regard to) s
5 443 M
( its DDP-eligibility. DDP-eligible data items MAY be removed from) s
5 432 M
( these special chunks and conveyed via normal chunks, but non-eligible) s
5 421 M
( data items MUST NOT appear in normal chunks.) s
5 399 M
(5. RPC-Over-RDMA In Operation) s
5 377 M
( Every RPC-over-RDMA Version One message has a header that includes a) s
5 366 M
( copy of the message's transaction ID, data for managing RDMA flow) s
5 355 M
( control credits, and lists of RDMA segments used for RDMA Read and) s
5 344 M
( Write operations. All RPC-over-RDMA header content is contained in) s
5 333 M
( the Transport stream, and thus MUST be XDR encoded.) s
5 311 M
( RPC message layout is unchanged from that described in [RFC5531]) s
5 300 M
( except for the possible reduction of data items that are moved by) s
5 289 M
( RDMA Read or Write operations.) s
5 267 M
( The RPC-over-RDMA protocol passes RPC messages without regard to) s
5 256 M
( their type \(CALL or REPLY\) or direction \(forwards or backwards\).) s
5 245 M
( Both endpoints of a connection MAY send any RPC-over-RDMA message) s
5 234 M
( header type at any time \(subject to credit limits\).) s
5 212 M
(5.1. XDR Protocol Definition) s
5 190 M
( This section contains a description of the core features of the RPC-) s
5 179 M
( over-RDMA Version One protocol, expressed in the XDR language) s
5 168 M
( [RFC4506].) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 22]) s
_R
S
%%Page: (23) 23
%%BeginPageSetup
_S
76 24 translate
/pagenum 23 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( This description is provided in a way that makes it simple to extract) s
5 674 M
( into ready-to-compile form. The reader can apply the following shell) s
5 663 M
( script to this document to produce a machine-readable XDR description) s
5 652 M
( of the RPC-over-RDMA Version One protocol without any OPTIONAL) s
5 641 M
( extensions.) s
5 608 M
( <CODE BEGINS>) s
5 586 M
( #!/bin/sh) s
5 575 M
( grep '^ *///' | sed 's?^ /// ??' | sed 's?^ *///$??') s
5 553 M
( <CODE ENDS>) s
5 520 M
( That is, if the above script is stored in a file called "extract.sh") s
5 509 M
( and this document is in a file called "spec.txt" then the reader can) s
5 498 M
( do the following to extract an XDR description file:) s
5 465 M
( <CODE BEGINS>) s
5 443 M
( sh extract.sh < spec.txt > rpcrdma_corev1.x) s
5 421 M
( <CODE ENDS>) s
5 388 M
( As described in Section 9.4, extensions to RPC-over-RDMA Version One,) s
5 377 M
( published as Proposed Standards, will have similar means of providing) s
5 366 M
( an XDR description appropriate to those extensions. Once XDR for) s
5 355 M
( extensions is also extracted, it can be appended to the XDR) s
5 344 M
( description file extracted from this document to produce a) s
5 333 M
( consolidated XDR description file reflecting all extensions selected) s
5 322 M
( for an RPC-over-RDMA implementation.) s
5 300 M
( RPC-over-RDMA is not a stand-alone RPC Program. To enable protocol) s
5 289 M
( extension, there is no single XDR entity which describes the format) s
5 278 M
( of RPC-over-RDMA headers. Instead, implementers need to follow the) s
5 267 M
( instructions in Section 5.1.4 to appropriately encode and decode) s
5 256 M
( protocol messages.) s
5 234 M
(5.1.1. Code Component License) s
5 212 M
( Code components extracted from this document must include the) s
5 201 M
( following license text. When the extracted XDR code is combined with) s
5 190 M
( other complementary XDR code which itself has an identical license,) s
5 179 M
( only a single copy of the license text need be preserved.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 23]) s
_R
S
%%Page: (24) 24
%%BeginPageSetup
_S
76 24 translate
/pagenum 24 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( <CODE BEGINS>) s
5 663 M
( /// /*) s
5 652 M
( /// * Copyright \(c\) 2010, 2015 IETF Trust and the persons) s
5 641 M
( /// * identified as authors of the code. All rights reserved.) s
5 630 M
( /// *) s
5 619 M
( /// * The authors of the code are:) s
5 608 M
( /// * B. Callaghan, T. Talpey, C. Lever, and D. Noveck.) s
5 597 M
( /// *) s
5 586 M
( /// * Redistribution and use in source and binary forms, with) s
5 575 M
( /// * or without modification, are permitted provided that the) s
5 564 M
( /// * following conditions are met:) s
5 553 M
( /// *) s
5 542 M
( /// * - Redistributions of source code must retain the above) s
5 531 M
( /// * copyright notice, this list of conditions and the) s
5 520 M
( /// * following disclaimer.) s
5 509 M
( /// *) s
5 498 M
( /// * - Redistributions in binary form must reproduce the above) s
5 487 M
( /// * copyright notice, this list of conditions and the) s
5 476 M
( /// * following disclaimer in the documentation and/or other) s
5 465 M
( /// * materials provided with the distribution.) s
5 454 M
( /// *) s
5 443 M
( /// * - Neither the name of Internet Society, IETF or IETF) s
5 432 M
( /// * Trust, nor the names of specific contributors, may be) s
5 421 M
( /// * used to endorse or promote products derived from this) s
5 410 M
( /// * software without specific prior written permission.) s
5 399 M
( /// *) s
5 388 M
( /// * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS) s
5 377 M
( /// * AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED) s
5 366 M
( /// * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE) s
5 355 M
( /// * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS) s
5 344 M
( /// * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO) s
5 333 M
( /// * EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE) s
5 322 M
( /// * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,) s
5 311 M
( /// * EXEMPLARY, OR CONSEQUENTIAL DAMAGES \(INCLUDING, BUT) s
5 300 M
( /// * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR) s
5 289 M
( /// * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS) s
5 278 M
( /// * INTERRUPTION\) HOWEVER CAUSED AND ON ANY THEORY OF) s
5 267 M
( /// * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,) s
5 256 M
( /// * OR TORT \(INCLUDING NEGLIGENCE OR OTHERWISE\) ARISING) s
5 245 M
( /// * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF) s
5 234 M
( /// * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.) s
5 223 M
( /// */) s
5 201 M
( <CODE ENDS>) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 24]) s
_R
S
%%Page: (25) 25
%%BeginPageSetup
_S
76 24 translate
/pagenum 25 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
(5.1.2. XDR Applying To All Versions Of RPC-Over-RDMA) s
5 663 M
( XDR data items defined in this section describe elements of the RPC-) s
5 652 M
( over-RDMA protocol that are not subject to change in subsequent) s
5 641 M
( versions. A full discussion of the extensibility model is in) s
5 630 M
( Section 9.) s
5 597 M
( <CODE BEGINS>) s
5 575 M
( /// typedef uint32 rdma_htype;) s
5 564 M
( ///) s
5 553 M
( /// struct rpcrdma_prefix {) s
5 542 M
( /// uint32 rdma_xid;) s
5 531 M
( /// uint32 rdma_version;) s
5 520 M
( /// uint32 rdma_credits;) s
5 509 M
( /// rpcrdma_htype rdma_htype;) s
5 498 M
( /// };) s
5 487 M
( ///) s
5 476 M
( /// /*) s
5 465 M
( /// * Mandatory RPC-over-RDMA message header types) s
5 454 M
( /// */) s
5 443 M
( /// const RDMA_MSG = 0;) s
5 432 M
( /// const RDMA_NOMSG = 1;) s
5 421 M
( /// const RDMA_ERROR = 4;) s
5 410 M
( ///) s
5 399 M
( /// struct rpcrdma_err_vers {) s
5 388 M
( /// uint32 rdma_vers_low;) s
5 377 M
( /// uint32 rdma_vers_high;) s
5 366 M
( /// };) s
5 344 M
( <CODE ENDS>) s
5 311 M
(5.1.3. XDR Applying To Version One Of RPC-Over-RDMA) s
5 289 M
( XDR data items defined in this section are subject to change in) s
5 278 M
( subsequent RPC-over-RDMA versions.) s
5 256 M
( Even though the names of structures and unions begin "rpcrdma1_") s
5 245 M
( these are not restricted to use in RPC-over-RDMA Version One.) s
5 234 M
( Structure definitions may be carried over unchanged to subsequence) s
5 223 M
( versions, but unions are subject to extension according to the rules) s
5 212 M
( for compatible XDR extension as discussed in Section 9. Comments) s
5 201 M
( identify items that cannot be changed in subsequent versions.) s
5 168 M
( <CODE BEGINS>) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 25]) s
_R
S
%%Page: (26) 26
%%BeginPageSetup
_S
76 24 translate
/pagenum 26 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( /// /*) s
5 674 M
( /// * Version One reserved message types) s
5 663 M
( /// */) s
5 652 M
( /// const RDMA_MSGP = 2;) s
5 641 M
( /// const RDMA_DONE = 3;) s
5 630 M
( ///) s
5 619 M
( /// struct rpcrdma1_segment {) s
5 608 M
( /// uint32 rdma_handle;) s
5 597 M
( /// uint32 rdma_length;) s
5 586 M
( /// uint64 rdma_offset;) s
5 575 M
( /// };) s
5 564 M
( ///) s
5 553 M
( /// struct rpcrdma1_read_segment {) s
5 542 M
( /// uint32 rdma_position;) s
5 531 M
( /// struct rpcrdma1_segment rdma_target;) s
5 520 M
( /// };) s
5 509 M
( ///) s
5 498 M
( /// struct rpcrdma1_read_list {) s
5 487 M
( /// struct rpcrdma1_read_segment rdma_entry;) s
5 476 M
( /// struct rpcrdma1_read_list *rdma_next;) s
5 465 M
( /// };) s
5 454 M
( ///) s
5 443 M
( /// struct rpcrdma1_write_chunk {) s
5 432 M
( /// struct rpcrdma1_segment rdma_target<>;) s
5 421 M
( /// };) s
5 410 M
( ///) s
5 399 M
( /// struct rpcrdma1_write_list {) s
5 388 M
( /// struct rpcrdma1_write_chunk rdma_entry;) s
5 377 M
( /// struct rpcrdma1_write_list *rdma_next;) s
5 366 M
( /// };) s
5 355 M
( ///) s
5 344 M
( /// struct rpcrdma1_chunks {) s
5 333 M
( /// struct rpcrdma1_read_list *rdma_reads;) s
5 322 M
( /// struct rpcrdma1_write_list *rdma_writes;) s
5 311 M
( /// struct rpcrdma1_write_chunk *rdma_reply;) s
5 300 M
( /// };) s
5 289 M
( ///) s
5 278 M
( /// struct rpcrdma1_padded {) s
5 267 M
( /// uint32 rdma_align;) s
5 256 M
( /// uint32 rdma_thresh;) s
5 245 M
( /// rpcrdma1_chunks rdma_chunks;) s
5 234 M
( /// };) s
5 223 M
( ///) s
5 212 M
( /// enum rpcrdma1_errcode {) s
5 201 M
( /// RDMA_ERR_VERS = 1,) s
5 190 M
( /// RDMA_ERR_BADHEADER = 2) s
5 179 M
( /// };) s
5 168 M
( ///) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 26]) s
_R
S
%%Page: (27) 27
%%BeginPageSetup
_S
76 24 translate
/pagenum 27 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( /// union rpcrdma1_error switch \(rpcrdma1_errcode rdma_err\) {) s
5 674 M
( /// case RDMA_ERR_VERS:) s
5 663 M
( /// rpcrdma_err_vers rdma_vrange; /* Immutable */) s
5 652 M
( /// case RDMA_ERR_BADHEADER:) s
5 641 M
( /// void;) s
5 630 M
( /// };) s
5 608 M
( <CODE ENDS>) s
5 575 M
(5.1.4. Use Of XDR Descriptions) s
5 553 M
( Though it is described by XDR, RPC-over-RDMA is not an RPC Program.) s
5 542 M
( Certain functions normally provided by RPC need to be addressed by) s
5 531 M
( the RPC-over-RDMA definition itself. In particular, the following) s
5 520 M
( functions normally provided by RPC need to be provided for as part of) s
5 509 M
( the RPC-over-RDMA XDR description:) s
5 487 M
( o negotiation of RPC-over-RDMA protocol version) s
5 465 M
( o Identifying RPC-over-RDMA header types that are followed by a) s
5 454 M
( Payload stream) s
5 432 M
( In [RFC5666] the XDR description did not take account of the natural) s
5 421 M
( layering between the part of RPC-over-RDMA functionality that) s
5 410 M
( performed RPC-layer like functions described above and that which) s
5 399 M
( implemented individual transport functions. As a result:) s
5 377 M
( o The four 32-bit words which must be the same in all versions of) s
5 366 M
( RPC-over-RDMA are split, with three of those words in struct) s
5 355 M
( rpcrdma1_header and the remaining word part of union) s
5 344 M
( rpcrdma1_body, together with each of the message bodies.) s
5 322 M
( o It is impossible, within the resulting structure, to add a new) s
5 311 M
( message type without modifying the existing XDR description.) s
5 289 M
( The XDR description introduced in this document reorganizes the XDR) s
5 278 M
( in line with this natural layering, while maintaining over-the-wire) s
5 267 M
( equivalence. As a result, the 32-bit big-endian field strating) s
5 256 M
( twelve bytes into the header is no longer the discriminator field of) s
5 245 M
( union rpcrdma1_body. Instead it is the last 32-bit word within) s
5 234 M
( struct rpcrdma_header which define the common \(i.e., for all RPC-) s
5 223 M
( over-RDMA versions\) header prefix. It retains its role of indicating) s
5 212 M
( the message type and deciding which particular header body is to) s
5 201 M
( follow.) s
5 179 M
( As a result there is no longer a single XDR item that encompasses the) s
5 168 M
( entire RPC-over-RDMA header. Instead, each RPC-over-RDMA meassage) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 27]) s
_R
S
%%Page: (28) 28
%%BeginPageSetup
_S
76 24 translate
/pagenum 28 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( consists of up to three items and those using XDR encode and decode) s
5 674 M
( must be aware that they proceed in sequence as follows:) s
5 652 M
( 1. A struct rpcrdma_prefix) s
5 630 M
( 2. Depending on the rdma_which field of the prefix, the appropriate) s
5 619 M
( header body for that message type as given by Table 1. In cases) s
5 608 M
( in which there is an undefined header type, this is to be treated) s
5 597 M
( as an XDR encode/decode error.) s
5 575 M
( 3. If allowed for that header type as defined in Table 1, an XDR) s
5 564 M
( stream for the RPC message being transported) s
5 531 M
( +--------------+------------------------+-------------------+) s
5 520 M
( | Message Type | Body | Payload stream? |) s
5 509 M
( +--------------+------------------------+-------------------+) s
5 498 M
( | RDMA_MSG | struct rpcrdma1_chunks | Yes |) s
5 487 M
( +--------------+------------------------+-------------------+) s
5 476 M
( | RDMA_NOMSG | struct rpcrdma1_chunks | No |) s
5 465 M
( +--------------+------------------------+-------------------+) s
5 454 M
( | RDMA_ERROR | union rpcrdma1_error | No |) s
5 443 M
( +--------------+------------------------+-------------------+) s
5 410 M
( Table 1. Header Type Characteristics) s
5 388 M
(5.2. Fixed Header Fields) s
5 366 M
( The RPC-over-RDMA header begins with four fixed 32-bit fields that) s
5 355 M
( control the RDMA interaction. These four fields, which must remain) s
5 344 M
( with the same meanings and in the same positions in all subsequent) s
5 333 M
( versions of the RPC-over-RDMA protocol, are described below.) s
5 300 M
( +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+) s
5 289 M
( | XID |) s
5 278 M
( +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+) s
5 267 M
( | Version Number |) s
5 256 M
( +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+) s
5 245 M
( | Credit Value |) s
5 234 M
( +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+) s
5 223 M
( | Procedure Number |) s
5 212 M
( +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 28]) s
_R
S
%%Page: (29) 29
%%BeginPageSetup
_S
76 24 translate
/pagenum 29 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
(5.2.1. Transaction ID \(XID\)) s
5 663 M
( The XID generated for the RPC Call and Reply. Having the XID at a) s
5 652 M
( fixed location in the header makes it easy for the receiver to) s
5 641 M
( establish context as soon as each RPC-over-RDMA message arrives.) s
5 630 M
( This XID MUST be the same as the XID in the RPC message. The) s
5 619 M
( receiver MAY perform its processing based solely on the XID in the) s
5 608 M
( RPC-over-RDMA header, and thereby ignore the XID in the RPC message,) s
5 597 M
( if it so chooses.) s
5 575 M
(5.2.2. Version Number) s
5 553 M
( For RPC-over-RDMA Version One, this field MUST contain the value one) s
5 542 M
( \(1\). Rules regarding changes to this transport protocol version) s
5 531 M
( number can be found in Section 9.3.) s
5 509 M
(5.2.3. Credit Value) s
5 487 M
( When sent in an RPC Call message, the requested credit value is) s
5 476 M
( provided. When sent in an RPC Reply message, the granted credit) s
5 465 M
( value is returned. RPC Calls SHOULD NOT be sent in excess of the) s
5 454 M
( currently granted limit. Further discussion of how the credit value) s
5 443 M
( is determined can be found in Section 4.3.) s
5 421 M
(5.2.4. Procedure number) s
5 399 M
( o RDMA_MSG = 0 indicates that chunk lists and a Payload stream) s
5 388 M
( follow. The format of the chunk lists is discussed below.) s
5 366 M
( o RDMA_NOMSG = 1 indicates that after the chunk lists there is no) s
5 355 M
( Payload stream. In this case, the chunk lists provide information) s
5 344 M
( to allow the responder to transfer the Payload stream using RDMA) s
5 333 M
( Read or Write operations.) s
5 311 M
( o RDMA_MSGP = 2 is reserved.) s
5 289 M
( o RDMA_DONE = 3 is reserved.) s
5 267 M
( o RDMA_ERROR = 4 is used to signal an encoding error in the RPC-) s
5 256 M
( over-RDMA header.) s
5 234 M
( An RDMA_MSG procedure conveys the Transport stream and the Payload) s
5 223 M
( stream via an RDMA Send operation. The Transport stream contains the) s
5 212 M
( four fixed fields, followed by the Read and Write lists and the Reply) s
5 201 M
( chunk, though any or all three MAY be marked as not present. The) s
5 190 M
( Payload stream then follows, beginning with its XID field. If a Read) s
5 179 M
( or Write chunk list is present, a portion of the Payload stream has) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 29]) s
_R
S
%%Page: (30) 30
%%BeginPageSetup
_S
76 24 translate
/pagenum 30 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( been excised and is conveyed separately via RDMA Read or Write) s
5 674 M
( operations.) s
5 652 M
( An RDMA_NOMSG procedure conveys the Transport stream via an RDMA Send) s
5 641 M
( operation. The Transport stream contains the four fixed fields,) s
5 630 M
( followed by the Read and Write chunk lists and the Reply chunk.) s
5 619 M
( Though any of these MAY be marked as not present, one MUST be present) s
5 608 M
( and MUST hold the Payload stream for this RPC-over-RDMA message. If) s
5 597 M
( a Read or Write chunk list is present, a portion of the Payload) s
5 586 M
( stream has been excised and is conveyed separately via RDMA Read or) s
5 575 M
( Write operations.) s
5 553 M
( An RDMA_ERROR procedure conveys the Transport stream via an RDMA Send) s
5 542 M
( operation. The Transport stream contains the four fixed fields,) s
5 531 M
( followed by formatted error information. No Payload stream is) s
5 520 M
( conveyed in this type of RPC-over-RDMA message.) s
5 498 M
( A gather operation on each RDMA Send operation can be used to combine) s
5 487 M
( the Transport and Payload streams, which might have been constructed) s
5 476 M
( in separate buffers. However, the total length of the gathered send) s
5 465 M
( buffers MUST NOT exceed the peer receiver's inline threshold.) s
5 443 M
(5.3. Chunk Lists) s
5 421 M
( The chunk lists in an RPC-over-RDMA Version One header are three XDR) s
5 410 M
( optional-data fields that follow the fixed header fields in RDMA_MSG) s
5 399 M
( and RDMA_NOMSG procedures. Read Section 4.19 of [RFC4506] carefully) s
5 388 M
( to understand how optional-data fields work. Examples of XDR encoded) s
5 377 M
( chunk lists are provided in Section 5.7 as an aid to understanding.) s
5 355 M
(5.3.1. Read List) s
5 333 M
( Each RDMA_MSG or RDMA_NOMSG procedure has one "Read list." The Read) s
5 322 M
( list is a list of zero or more Read segments, provided by the) s
5 311 M
( requester, that are grouped by their Position fields into Read) s
5 300 M
( chunks. Each Read chunk advertises the location of argument data the) s
5 289 M
( responder is to retrieve via RDMA Read operations. The requester has) s
5 278 M
( removed the data in these chunks from the call's Payload stream.) s
5 256 M
( Via a Position Zero Read Chunk, a requester may provide an RPC Call) s
5 245 M
( message as a chunk in the Read list.) s
5 223 M
( If the RPC Call has no argument data that is DDP-eligible and the) s
5 212 M
( Position Zero Read Chunk is not being used, the requester leaves the) s
5 201 M
( Read list empty.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 30]) s
_R
S
%%Page: (31) 31
%%BeginPageSetup
_S
76 24 translate
/pagenum 31 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
(5.3.2. Write List) s
5 663 M
( Each RDMA_MSG or RDMA_NOMSG procedure has one "Write list." The) s
5 652 M
( Write list is a list of zero or more Write chunks, provided by the) s
5 641 M
( requester. Each Write chunk is an array of RDMA segments, thus the) s
5 630 M
( Write list is a list of counted arrays. Each Write chunk advertises) s
5 619 M
( receptacles for DDP-eligible data to be pushed by the responder via) s
5 608 M
( RDMA Write operations. If the RPC Reply has no possible DDP-eligible) s
5 597 M
( result data items, the requester leaves the Write list empty.) s
5 575 M
( *** This section needs to specify when a requester must provide Write) s
5 564 M
( chunks, and how many chunks must be provided. ***) s
5 542 M
( When a Write list is provided for the results of an RPC Call, the) s
5 531 M
( responder MUST provide data corresponding to DDP-eligible XDR data) s
5 520 M
( items via RDMA Write operations to the memory referenced in the Write) s
5 509 M
( list. The responder removes the data in these chunks from the) s
5 498 M
( reply's Payload stream.) s
5 476 M
( When multiple Write chunks are present, the responder fills in each) s
5 465 M
( Write chunk with a DDP-eligible result until either there are no more) s
5 454 M
( results or no more Write chunks. The requester may not be able to) s
5 443 M
( predict which DDP-eligible data item goes in which chunk. Thus the) s
5 432 M
( requester is responsible for allocating and registering Write chunks) s
5 421 M
( large enough to accommodate the largest XDR data item that might be) s
5 410 M
( associated with each chunk in the list.) s
5 388 M
( The RPC Reply conveys the size of result data items by returning each) s
5 377 M
( Write chunk to the requester with the segment lengths rewritten to) s
5 366 M
( match the actual data transferred. Decoding the reply therefore) s
5 355 M
( performs no local data copying but merely returns the length obtained) s
5 344 M
( from the reply.) s
5 322 M
( Each decoded result consumes one entry in the Write list, which in) s
5 311 M
( turn consists of an array of RDMA segments. The length of a Write) s
5 300 M
( chunk is therefore the sum of all returned lengths in all segments) s
5 289 M
( comprising the corresponding list entry. As each Write chunk is) s
5 278 M
( decoded, the entire Write list entry is consumed.) s
5 256 M
( A requester constructs the Write list for an RPC transaction before) s
5 245 M
( the responder has formulated its reply. When there is only one DDP-) s
5 234 M
( eligible result data item, the requester inserts only a single Write) s
5 223 M
( chunk in the Write list. If the responder populates that chunk with) s
5 212 M
( data, the requester knows with certainty which result data item is) s
5 201 M
( contained in it.) s
5 179 M
( However, Upper Layer Protocol procedures may allow replies where more) s
5 168 M
( than one result data item is DDP-eligible. For example, an NFSv4) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 31]) s
_R
S
%%Page: (32) 32
%%BeginPageSetup
_S
76 24 translate
/pagenum 32 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( COMPOUND procedure is composed of individual NFSv4 operations, more) s
5 674 M
( than one of which may have a reply containing a DDP-eligible result.) s
5 652 M
( As stated above, when multiple Write chunks are present, the) s
5 641 M
( responder reduces DDP-eligible result until either there are no more) s
5 630 M
( results or no more Write chunks. Then, as the requester decodes the) s
5 619 M
( reply Payload stream, it is clear from the contents of the reply) s
5 608 M
( which Write chunk contains which data item.) s
5 586 M
(5.3.3. Reply Chunk) s
5 564 M
( Each RDMA_MSG or RDMA_NOMSG procedure has one "Reply chunk." The) s
5 553 M
( Reply chunk is a Write chunk, provided by the requester. The Reply) s
5 542 M
( chunk is a single counted array of RDMA segments.) s
5 520 M
( A requester MUST provide a Reply chunk whenever the maximum possible) s
5 509 M
( size of the reply is larger than its own inline threshold. The Reply) s
5 498 M
( chunk MUST be large enough to contain a Payload stream \(RPC message\)) s
5 487 M
( of this maximum size. If the actual reply Payload stream is smaller) s
5 476 M
( than the requester's inline threshold, the responder MAY return it as) s
5 465 M
( a Short message rather than using the Reply chunk.) s
5 443 M
(5.4. Memory Registration) s
5 421 M
( RDMA requires that data is transferred between only registered memory) s
5 410 M
( segments at the source and destination. All protocol headers as well) s
5 399 M
( as separately transferred data chunks must reside in registered) s
5 388 M
( memory.) s
5 366 M
( Since the cost of registering and de-registering memory can be a) s
5 355 M
( significant proportion of the RDMA transaction cost, it is important) s
5 344 M
( to minimize registration activity. For memory that is targeted by) s
5 333 M
( RDMA Send and Receive operations, a local-only registration is) s
5 322 M
( sufficient and can be left in place during the life of a connection) s
5 311 M
( without any risk of data exposure.) s
5 289 M
(5.4.1. Registration Longevity) s
5 267 M
( Data transferred via RDMA Read and Write can reside in a memory) s
5 256 M
( allocation not in the control of the RPC-over-RDMA transport. These) s
5 245 M
( memory allocations can persist outside the bounds of an RPC) s
5 234 M
( transaction. They are registered and invalidated as needed, as part) s
5 223 M
( of each RPC transaction.) s
5 201 M
( The requester endpoint must ensure that memory segments associated) s
5 190 M
( with each RPC transaction are properly fenced from responders before) s
5 179 M
( allowing Upper Layer access to the data contained in them. Moreover,) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 32]) s
_R
S
%%Page: (33) 33
%%BeginPageSetup
_S
76 24 translate
/pagenum 33 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( the requester must not access these memory segments while the) s
5 674 M
( responder has access to them.) s
5 652 M
( This includes segments that are associated with canceled RPCs. A) s
5 641 M
( responder cannot know that the requester is no longer waiting for a) s
5 630 M
( reply, and might proceed to read or even update memory that the) s
5 619 M
( requester might have released for other use.) s
5 597 M
(5.4.2. Communicating DDP-Eligibility) s
5 575 M
( The interface by which an Upper Layer Protocol implementation) s
5 564 M
( communicates the eligibility of a data item locally to its local RPC-) s
5 553 M
( over-RDMA endpoint is not described by this specification.) s
5 531 M
( Depending on the implementation and constraints imposed by Upper) s
5 520 M
( Layer Bindings, it is possible to implement reduction transparently) s
5 509 M
( to upper layers. Such implementations may lead to inefficiencies,) s
5 498 M
( either because they require the RPC layer to perform expensive) s
5 487 M
( registration and de-registration of memory "on the fly", or they may) s
5 476 M
( require using RDMA chunks in reply messages, along with the resulting) s
5 465 M
( additional handshaking with the RPC-over-RDMA peer.) s
5 443 M
( However, these issues are internal and generally confined to the) s
5 432 M
( local interface between RPC and its upper layers, one in which) s
5 421 M
( implementations are free to innovate. The only requirement is that) s
5 410 M
( the resulting RPC-over-RDMA protocol sent to the peer is valid for) s
5 399 M
( the upper layer.) s
5 377 M
(5.4.3. Registration Strategies) s
5 355 M
( The choice of which memory registration strategies to employ is left) s
5 344 M
( to requester and responder implementers. To support the widest array) s
5 333 M
( of RDMA implementations, as well as the most general steering tag) s
5 322 M
( scheme, an Offset field is included in each segment.) s
5 300 M
( While zero-based offset schemes are available in many RDMA) s
5 289 M
( implementations, their use by RPC requires individual registration of) s
5 278 M
( each segment. For such implementations, this can be a significant) s
5 267 M
( overhead. By providing an offset in each chunk, many pre-) s
5 256 M
( registration or region-based registrations can be readily supported.) s
5 245 M
( By using a single, universal chunk representation, the RPC-over-RDMA) s
5 234 M
( protocol implementation is simplified to its most general form.) s
5 212 M
(5.5. Error Handling) s
5 190 M
( A receiver performs basic validity checks on the RPC-over-RDMA header) s
5 179 M
( and chunk contents before it passes the RPC message to the RPC) s
5 168 M
( consumer. If errors are detected in an RPC-over-RDMA header, an) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 33]) s
_R
S
%%Page: (34) 34
%%BeginPageSetup
_S
76 24 translate
/pagenum 34 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( RDMA_ERROR procedure MUST be generated. Because the transport layer) s
5 674 M
( may not be aware of the direction of a problematic RPC message, an) s
5 663 M
( RDMA_ERROR procedure MAY be generated by either a requester or a) s
5 652 M
( responder.) s
5 630 M
( To form an RDMA_ERROR procedure: The rdma_xid field MUST contain the) s
5 619 M
( same XID that was in the rdma_xid field in the failing request; The) s
5 608 M
( rdma_vers field MUST contain the same version that was in the) s
5 597 M
( rdma_vers field in the failing request; The rdma_proc field MUST) s
5 586 M
( contain the value RDMA_ERROR; The rdma_err field contains a value) s
5 575 M
( that reflects the type of error that occurred, as described below.) s
5 553 M
( An RDMA_ERROR procedure indicates a permanent error. Receipt of this) s
5 542 M
( procedure completes the RPC transaction associated with XID in the) s
5 531 M
( rdma_xid field. A receiver MUST silently discard an RDMA_ERROR) s
5 520 M
( procedure that cannot be decoded.) s
5 498 M
(5.5.1. Header Version Mismatch) s
5 476 M
( When a receiver detects an RPC-over-RDMA header version that it does) s
5 465 M
( not support \(currently this document defines only Version One\), it) s
5 454 M
( MUST reply with an RDMA_ERROR procedure and set the rdma_err value to) s
5 443 M
( RDMA_ERR_VERS, also providing the low and high inclusive version) s
5 432 M
( numbers it does, in fact, support.) s
5 410 M
(5.5.2. XDR Errors) s
5 388 M
( A receiver might encounter an XDR parsing error that prevents it from) s
5 377 M
( processing the incoming Transport stream. Examples of such errors) s
5 366 M
( include an invalid value in the rdma_proc field, an RDMA_NOMSG) s
5 355 M
( message that has no chunk lists, or the contents of the rdma_xid) s
5 344 M
( field might not match the contents of the XID field in the) s
5 333 M
( accompanying RPC message. If the rdma_vers field contains a) s
5 322 M
( recognized value, but an XDR parsing error occurs, the responder MUST) s
5 311 M
( reply with an RDMA_ERROR procedure and set the rdma_err value to) s
5 300 M
( RDMA_ERR_BADHEADER.) s
5 278 M
( When a responder receives a valid RPC-over-RDMA header but the) s
5 267 M
( responder's Upper Layer Protocol implementation cannot parse the RPC) s
5 256 M
( arguments in the RPC Call message, the responder SHOULD return a) s
5 245 M
( RPC_GARBAGEARGS reply, using an RDMA_MSG procedure. This type of) s
5 234 M
( parsing failure might be due to mismatches between chunk sizes or) s
5 223 M
( offsets and the contents of the Payload stream, for example. A) s
5 212 M
( responder MAY also report the presence of a non-DDP-eligible data) s
5 201 M
( item in a Read or Write chunk using RPC_GARBAGEARGS.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 34]) s
_R
S
%%Page: (35) 35
%%BeginPageSetup
_S
76 24 translate
/pagenum 35 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
(5.5.3. Responder RDMA Operational Errors) s
5 663 M
( In RPC-over-RDMA Version One, it is the responder which drives RDMA) s
5 652 M
( Read and Write operations that target the requester's memory.) s
5 641 M
( Problems might arise as the responder attempts to use requester-) s
5 630 M
( provided resources for RDMA operations. For example:) s
5 608 M
( o Chunks can be validated only by using their contents to form RDMA) s
5 597 M
( Read or Write operations. If chunk contents are invalid \(say, a) s
5 586 M
( segment is no longer registered, or a chunk length is too long\), a) s
5 575 M
( Remote Access error occurs.) s
5 553 M
( o If a requester's receive buffer is too small, the responder's Send) s
5 542 M
( operation completes with a Local Length Error.) s
5 520 M
( o If the requester-provided Reply chunk is too small to accommodate) s
5 509 M
( a large RPC Reply, a Remote Access error occurs. A responder can) s
5 498 M
( detect this problem before attempting to write past the end of the) s
5 487 M
( Reply chunk.) s
5 465 M
( RDMA operational errors are typically fatal to the connection. To) s
5 454 M
( avoid a retransmission loop and repeated connection loss that) s
5 443 M
( deadlocks the connection, once the requester has re-established a) s
5 432 M
( connection, the responder should send an RDMA_ERROR reply with an) s
5 421 M
( rdma_err value of RDMA_ERR_BADHEADER to indicate that no RPC-level) s
5 410 M
( reply is possible for that XID.) s
5 388 M
(5.5.4. Other Operational Errors) s
5 366 M
( While a requester is constructing a Call message, an unrecoverable) s
5 355 M
( problem might occur that prevents the requester from posting further) s
5 344 M
( RDMA Work Requests on behalf of that message. As with other) s
5 333 M
( transports, if a requester is unable to construct and transmit a Call) s
5 322 M
( message, the associated RPC transaction fails immediately.) s
5 300 M
( After a requester has received a reply, if it is unable to invalidate) s
5 289 M
( a memory region due to an unrecoverable problem, the requester MUST) s
5 278 M
( close the connection to fence that memory from the responder before) s
5 267 M
( the associated RPC transaction is complete.) s
5 245 M
( While a responder is constructing a Reply message or error message,) s
5 234 M
( an unrecoverable problem might occur that prevents the responder from) s
5 223 M
( posting further RDMA Work Requests on behalf of that message. If a) s
5 212 M
( responder is unable to construct and transmit a Reply or error) s
5 201 M
( message, the responder MUST close the connection to signal to the) s
5 190 M
( requester that a reply was lost.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 35]) s
_R
S
%%Page: (36) 36
%%BeginPageSetup
_S
76 24 translate
/pagenum 36 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
(5.5.5. RDMA Transport Errors) s
5 663 M
( The RDMA connection and physical link provide some degree of error) s
5 652 M
( detection and retransmission. iWARP's Marker PDU Aligned \(MPA\) layer) s
5 641 M
( \(when used over TCP\), Stream Control Transmission Protocol \(SCTP\), as) s
5 630 M
( well as the InfiniBand link layer all provide Cyclic Redundancy Check) s
5 619 M
( \(CRC\) protection of the RDMA payload, and CRC-class protection is a) s
5 608 M
( general attribute of such transports.) s
5 586 M
( Additionally, the RPC layer itself can accept errors from the link) s
5 575 M
( level and recover via retransmission. RPC recovery can handle) s
5 564 M
( complete loss and re-establishment of the link.) s
5 542 M
( The details of reporting and recovery from RDMA link layer errors are) s
5 531 M
( outside the scope of this protocol specification. See Section 10 for) s
5 520 M
( further discussion of the use of RPC-level integrity schemes to) s
5 509 M
( detect errors.) s
5 487 M
(5.6. Protocol Elements No Longer Supported) s
5 465 M
( The following protocol elements are no longer supported in RPC-over-) s
5 454 M
( RDMA Version One. Related enum values and structure definitions) s
5 443 M
( remain in the RPC-over-RDMA Version One protocol for backwards) s
5 432 M
( compatibility.) s
5 410 M
(5.6.1. RDMA_MSGP) s
5 388 M
( The specification of RDMA_MSGP in Section 3.9 of [RFC5666] is) s
5 377 M
( incomplete. To fully specify RDMA_MSGP would require:) s
5 355 M
( o Updating the definition of DDP-eligibility to include data items) s
5 344 M
( that may be transferred, with padding, via RDMA_MSGP procedures) s
5 322 M
( o Adding full operational descriptions of the alignment and) s
5 311 M
( threshold fields) s
5 289 M
( o Discussing how alignment preferences are communicated between two) s
5 278 M
( peers without using CCP) s
5 256 M
( o Describing the treatment of RDMA_MSGP procedures that convey Read) s
5 245 M
( or Write chunks) s
5 223 M
( The RDMA_MSGP message type is beneficial only when the padded data) s
5 212 M
( payload is at the end of an RPC message's argument or result list.) s
5 201 M
( This is not typical for NFSv4 COMPOUND RPCs, which often include a) s
5 190 M
( GETATTR operation as the final element of the compound operation) s
5 179 M
( array.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 36]) s
_R
S
%%Page: (37) 37
%%BeginPageSetup
_S
76 24 translate
/pagenum 37 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( Without a full specification of RDMA_MSGP, there has been no fully) s
5 674 M
( implemented prototype of it. Without a complete prototype of) s
5 663 M
( RDMA_MSGP support, it is difficult to assess whether this protocol) s
5 652 M
( element has benefit, or can even be made to work interoperably.) s
5 630 M
( Therefore, senders MUST NOT send RDMA_MSGP procedures. When) s
5 619 M
( receiving an RDMA_MSGP procedure, receivers SHOULD reply with an) s
5 608 M
( RDMA_ERROR procedure, setting the rdma_err field to) s
5 597 M
( RDMA_ERR_BADHEADER.) s
5 575 M
(5.6.2. RDMA_DONE) s
5 553 M
( Because no implementation of RPC-over-RDMA Version One uses the Read-) s
5 542 M
( Read transfer model, there is never a need to send an RDMA_DONE) s
5 531 M
( procedure.) s
5 509 M
( Therefore, senders MUST NOT send RDMA_DONE messages. When receiving) s
5 498 M
( an RDMA_DONE procedure, receivers SHOULD reply with an RDMA_ERROR) s
5 487 M
( procedure, setting the rdma_err field to RDMA_ERR_BADHEADER.) s
5 465 M
(5.7. XDR Examples) s
5 443 M
( RPC-over-RDMA chunk lists are complex data types. In this section,) s
5 432 M
( illustrations are provided to help readers grasp how chunk lists are) s
5 421 M
( represented inside an RPC-over-RDMA header.) s
5 399 M
( An RDMA segment is the simplest component, being made up of a 32-bit) s
5 388 M
( handle \(H\), a 32-bit length \(L\), and 64-bits of offset \(OO\). Once) s
5 377 M
( flattened into an XDR stream, RDMA segments appear as) s
5 344 M
( HLOO) s
5 311 M
( A Read segment has an additional 32-bit position field. Read) s
5 300 M
( segments appear as) s
5 267 M
( PHLOO) s
5 234 M
( A Read chunk is a list of Read segments. Each segment is preceded by) s
5 223 M
( a 32-bit word containing a one if there is a segment, or a zero if) s
5 212 M
( there are no more segments \(optional-data\). In XDR form, this would) s
5 201 M
( look like) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 37]) s
_R
S
%%Page: (38) 38
%%BeginPageSetup
_S
76 24 translate
/pagenum 38 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( 1 PHLOO 1 PHLOO 1 PHLOO 0) s
5 652 M
( where P would hold the same value for each segment belonging to the) s
5 641 M
( same Read chunk.) s
5 619 M
( The Read List is also a list of Read segments. In XDR form, this) s
5 608 M
( would look like a Read chunk, except that the P values could vary) s
5 597 M
( across the list. An empty Read List is encoded as a single 32-bit) s
5 586 M
( zero.) s
5 564 M
( One Write chunk is a counted array of segments. In XDR form, the) s
5 553 M
( count would appear as the first 32-bit word, followed by an HLOO for) s
5 542 M
( each element of the array. For instance, a Write chunk with three) s
5 531 M
( elements would look like) s
5 498 M
( 3 HLOO HLOO HLOO) s
5 465 M
( The Write List is a list of counted arrays. In XDR form, this is a) s
5 454 M
( combination of optional-data and counted arrays. To represent a) s
5 443 M
( Write List containing a Write chunk with three segments and a Write) s
5 432 M
( chunk with two segments, XDR would encode) s
5 399 M
( 1 3 HLOO HLOO HLOO 1 2 HLOO HLOO 0) s
5 366 M
( An empty Write List is encoded as a single 32-bit zero.) s
5 344 M
( The Reply chunk is a Write chunk. Since it is an optional-data) s
5 333 M
( field, however, there is a 32-bit field in front of it that contains) s
5 322 M
( a one if the Reply chunk is present, or a zero if it is not. After) s
5 311 M
( encoding, a Reply chunk with 2 segments would look like) s
5 278 M
( 1 2 HLOO HLOO) s
5 245 M
( Frequently a requester does not provide any chunks. In that case,) s
5 234 M
( after the four fixed fields in the RPC-over-RDMA header, there are) s
5 223 M
( simply three 32-bit fields that contain zero.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 38]) s
_R
S
%%Page: (39) 39
%%BeginPageSetup
_S
76 24 translate
/pagenum 39 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
(6. RPC Bind Parameters) s
5 663 M
( In setting up a new RDMA connection, the first action by a requester) s
5 652 M
( is to obtain a transport address for the responder. The mechanism) s
5 641 M
( used to obtain this address, and to open an RDMA connection is) s
5 630 M
( dependent on the type of RDMA transport, and is the responsibility of) s
5 619 M
( each RPC protocol binding and its local implementation.) s
5 597 M
( RPC services normally register with a portmap or rpcbind [RFC1833]) s
5 586 M
( service, which associates an RPC Program number with a service) s
5 575 M
( address. \(In the case of UDP or TCP, the service address for NFS is) s
5 564 M
( normally port 2049.\) This policy is no different with RDMA) s
5 553 M
( transports, although it may require the allocation of port numbers) s
5 542 M
( appropriate to each Upper Layer Protocol that uses the RPC framing) s
5 531 M
( defined here.) s
5 509 M
( When mapped atop the iWARP transport [RFC5040] [RFC5041], which uses) s
5 498 M
( IP port addressing due to its layering on TCP and/or SCTP, port) s
5 487 M
( mapping is trivial and consists merely of issuing the port in the) s
5 476 M
( connection process. The NFS/RDMA protocol service address has been) s
5 465 M
( assigned port 20049 by IANA, for both iWARP/TCP and iWARP/SCTP.) s
5 443 M
( When mapped atop InfiniBand [IB], which uses a Group Identifier) s
5 432 M
( \(GID\)-based service endpoint naming scheme, a translation MUST be) s
5 421 M
( employed. One such translation is defined in the InfiniBand Port) s
5 410 M
( Addressing Annex [IBPORT], which is appropriate for translating IP) s
5 399 M
( port addressing to the InfiniBand network. Therefore, in this case,) s
5 388 M
( IP port addressing may be readily employed by the upper layer.) s
5 366 M
( When a mapping standard or convention exists for IP ports on an RDMA) s
5 355 M
( interconnect, there are several possibilities for each upper layer to) s
5 344 M
( consider:) s
5 322 M
( o One possibility is to have responder register its mapped IP port) s
5 311 M
( with the rpcbind service, under the netid \(or netid's\) defined) s
5 300 M
( here. An RPC-over-RDMA-aware requester can then resolve its) s
5 289 M
( desired service to a mappable port, and proceed to connect. This) s
5 278 M
( is the most flexible and compatible approach, for those upper) s
5 267 M
( layers that are defined to use the rpcbind service.) s
5 245 M
( o A second possibility is to have the responder's portmapper) s
5 234 M
( register itself on the RDMA interconnect at a "well known" service) s
5 223 M
( address \(on UDP or TCP, this corresponds to port 111\). A) s
5 212 M
( requester could connect to this service address and use the) s
5 201 M
( portmap protocol to obtain a service address in response to a) s
5 190 M
( program number, e.g., an iWARP port number, or an InfiniBand GID.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 39]) s
_R
S
%%Page: (40) 40
%%BeginPageSetup
_S
76 24 translate
/pagenum 40 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( o Alternatively, the requester could simply connect to the mapped) s
5 674 M
( well-known port for the service itself, if it is appropriately) s
5 663 M
( defined. By convention, the NFS/RDMA service, when operating atop) s
5 652 M
( such an InfiniBand fabric, will use the same 20049 assignment as) s
5 641 M
( for iWARP.) s
5 619 M
( Historically, different RPC protocols have taken different approaches) s
5 608 M
( to their port assignment; therefore, the specific method is left to) s
5 597 M
( each RPC-over-RDMA-enabled Upper Layer binding, and not addressed) s
5 586 M
( here.) s
5 564 M
( In Section 11, this specification defines two new "netid" values, to) s
5 553 M
( be used for registration of upper layers atop iWARP [RFC5040]) s
5 542 M
( [RFC5041] and \(when a suitable port translation service is available\)) s
5 531 M
( InfiniBand [IB]. Additional RDMA-capable networks MAY define their) s
5 520 M
( own netids, or if they provide a port translation, MAY share the one) s
5 509 M
( defined here.) s
5 487 M
(7. Bi-Directional RPC-Over-RDMA) s
5 465 M
(7.1. RPC Direction) s
5 443 M
(7.1.1. Forward Direction) s
5 421 M
( A traditional ONC RPC client is always a requester. A traditional) s
5 410 M
( ONC RPC service is always a responder. This traditional form of ONC) s
5 399 M
( RPC message passing is referred to as operation in the "forward) s
5 388 M
( direction.") s
5 366 M
( During forward direction operation, the ONC RPC client is responsible) s
5 355 M
( for establishing transport connections.) s
5 333 M
(7.1.2. Backward Direction) s
5 311 M
( The ONC RPC standard does not forbid passing messages in the other) s
5 300 M
( direction. An ONC RPC service endpoint can act as a requester, in) s
5 289 M
( which case an ONC RPC client endpoint acts as a responder. This form) s
5 278 M
( of message passing is referred to as operation in the "backward) s
5 267 M
( direction.") s
5 245 M
( During backward direction operation, the ONC RPC client is) s
5 234 M
( responsible for establishing transport connections, even though ONC) s
5 223 M
( RPC Calls come from the ONC RPC server.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 40]) s
_R
S
%%Page: (41) 41
%%BeginPageSetup
_S
76 24 translate
/pagenum 41 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
(7.1.3. Bi-direction) s
5 663 M
( A pair of endpoints may choose to use only forward or only backward) s
5 652 M
( direction operations on a particular transport. Or, the endpoints) s
5 641 M
( may send operations in both directions concurrently on the same) s
5 630 M
( transport.) s
5 608 M
( Bi-directional operation occurs when both transport endpoints act as) s
5 597 M
( a requester and a responder at the same time. As above, the ONC RPC) s
5 586 M
( client is responsible for establishing transport connections.) s
5 564 M
(7.1.4. XIDs with Bi-direction) s
5 542 M
( During bi-directional operation, the forward and backward directions) s
5 531 M
( use independent xid spaces.) s
5 509 M
( In other words, a forward direction requester MAY use the same xid) s
5 498 M
( value at the same time as a backward direction requester on the same) s
5 487 M
( transport connection, but such concurrent requests represent distinct) s
5 476 M
( ONC RPC transactions.) s
5 454 M
(7.2. Backward Direction Flow Control) s
5 432 M
(7.2.1. Backward RPC-over-RDMA Credits) s
5 410 M
( Credits work the same way in the backward direction as they do in the) s
5 399 M
( forward direction. However, forward direction credits and backward) s
5 388 M
( direction credits are accounted separately.) s
5 366 M
( In other words, the forward direction credit value is the same) s
5 355 M
( whether or not there are backward direction resources associated with) s
5 344 M
( an RPC-over-RDMA transport connection. The backward direction credit) s
5 333 M
( value MAY be different than the forward direction credit value. The) s
5 322 M
( rdma_credit field in a backward direction RPC-over-RDMA message MUST) s
5 311 M
( NOT contain the value zero.) s
5 289 M
( A backward direction requester \(an RPC-over-RDMA service endpoint\)) s
5 278 M
( requests credits from the responder \(an RPC-over-RDMA client) s
5 267 M
( endpoint\). The responder reports how many credits it can grant.) s
5 256 M
( This is the number of backward direction Calls the responder is) s
5 245 M
( prepared to handle at once.) s
5 223 M
( When an RPC-over-RDMA server endpoint is operating correctly, it) s
5 212 M
( sends no more outstanding requests at a time than the client) s
5 201 M
( endpoint's advertised backward direction credit value.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 41]) s
_R
S
%%Page: (42) 42
%%BeginPageSetup
_S
76 24 translate
/pagenum 42 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
(7.2.2. Receive Buffer Management) s
5 663 M
( An RPC-over-RDMA transport endpoint must pre-post receive buffers) s
5 652 M
( before it can receive and process incoming RPC-over-RDMA messages.) s
5 641 M
( If a sender transmits a message for a receiver which has no posted) s
5 630 M
( receive buffer, the RDMA provider MAY drop the RDMA connection.) s
5 608 M
(7.2.2.1. Client Receive Buffers) s
5 586 M
( Typically an RPC-over-RDMA caller posts only as many receive buffers) s
5 575 M
( as there are outstanding RPC Calls. A client endpoint without) s
5 564 M
( backward direction support might therefore at times have no pre-) s
5 553 M
( posted receive buffers.) s
5 531 M
( To receive incoming backward direction Calls, an RPC-over-RDMA client) s
5 520 M
( endpoint must pre-post enough additional receive buffers to match its) s
5 509 M
( advertised backward direction credit value. Each outstanding forward) s
5 498 M
( direction RPC requires an additional receive buffer above this) s
5 487 M
( minimum.) s
5 465 M
( When an RDMA transport connection is lost, all active receive buffers) s
5 454 M
( are flushed and are no longer available to receive incoming messages.) s
5 443 M
( When a fresh transport connection is established, a client endpoint) s
5 432 M
( must re-post a receive buffer to handle the Reply for each) s
5 421 M
( retransmitted forward direction Call, and a full set of receive) s
5 410 M
( buffers to handle backward direction Calls.) s
5 388 M
(7.2.2.2. Server Receive Buffers) s
5 366 M
( A forward direction RPC-over-RDMA service endpoint posts as many) s
5 355 M
( receive buffers as it expects incoming forward direction Calls. That) s
5 344 M
( is, it posts no fewer buffers than the number of RPC-over-RDMA) s
5 333 M
( credits it advertises in the rdma_credit field of forward direction) s
5 322 M
( RPC replies.) s
5 300 M
( To receive incoming backward direction replies, an RPC-over-RDMA) s
5 289 M
( server endpoint must pre-post a receive buffer for each backward) s
5 278 M
( direction Call it sends.) s
5 256 M
( When the existing transport connection is lost, all active receive) s
5 245 M
( buffers are flushed and are no longer available to receive incoming) s
5 234 M
( messages. When a fresh transport connection is established, a server) s
5 223 M
( endpoint must re-post a receive buffer to handle the Reply for each) s
5 212 M
( retransmitted backward direction Call, and a full set of receive) s
5 201 M
( buffers for receiving forward direction Calls.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 42]) s
_R
S
%%Page: (43) 43
%%BeginPageSetup
_S
76 24 translate
/pagenum 43 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
(7.3. Conventions For Backward Operation) s
5 663 M
(7.3.1. In the Absense of Backward Direction Support) s
5 641 M
( An RPC-over-RDMA transport endpoint might not support backward) s
5 630 M
( direction operation. There might be no mechanism in the transport) s
5 619 M
( implementation to do so, or the Upper Layer Protocol consumer might) s
5 608 M
( not yet have configured the transport to handle backward direction) s
5 597 M
( traffic.) s
5 575 M
( A loss of the RDMA connection may result if the receiver is not) s
5 564 M
( prepared to receive an incoming message. Thus a denial-of-service) s
5 553 M
( could result if a sender continues to send backchannel messages after) s
5 542 M
( every transport reconnect to an endpoint that is not prepared to) s
5 531 M
( receive them.) s
5 509 M
( For RPC-over-RDMA Version One transports, the Upper Layer Protocol is) s
5 498 M
( responsible for informing its peer when it has established a backward) s
5 487 M
( direction capability. Otherwise even a simple backward direction) s
5 476 M
( NULL probe from a peer would result in a lost connection.) s
5 454 M
( An Upper Layer Protocol consumer MUST NOT perform backward direction) s
5 443 M
( ONC RPC operations unless the peer consumer has indicated it is) s
5 432 M
( prepared to handle them. A description of Upper Layer Protocol) s
5 421 M
( mechanisms used for this indication is outside the scope of this) s
5 410 M
( document.) s
5 388 M
(7.3.2. Backward Direction Retransmission) s
5 366 M
( In rare cases, an ONC RPC transaction cannot be completed within a) s
5 355 M
( certain time. This can be because the transport connection was lost,) s
5 344 M
( the Call or Reply message was dropped, or because the Upper Layer) s
5 333 M
( consumer delayed or dropped the ONC RPC request. Typically, the) s
5 322 M
( requester sends the transaction again, reusing the same RPC XID.) s
5 311 M
( This is known as an "RPC retransmission".) s
5 289 M
( In the forward direction, the Caller is the ONC RPC client. The) s
5 278 M
( client is always responsible for establishing a transport connection) s
5 267 M
( before sending again.) s
5 245 M
( In the backward direction, the Caller is the ONC RPC server. Because) s
5 234 M
( an ONC RPC server does not establish transport connections with) s
5 223 M
( clients, it cannot send a retransmission if there is no transport) s
5 212 M
( connection. It must wait for the ONC RPC client to re-establish the) s
5 201 M
( transport connection before it can retransmit ONC RPC transactions in) s
5 190 M
( the backward direction.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 43]) s
_R
S
%%Page: (44) 44
%%BeginPageSetup
_S
76 24 translate
/pagenum 44 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( If an ONC RPC client has no work to do, it may be some time before it) s
5 674 M
( re-establishes a transport connection. Backward direction Callers) s
5 663 M
( must be prepared to wait indefinitely before a connection is) s
5 652 M
( established before a pending backward direction ONC RPC Call can be) s
5 641 M
( retransmitted.) s
5 619 M
(7.3.3. Backward Direction Message Size) s
5 597 M
( RPC-over-RDMA backward direction messages are transmitted and) s
5 586 M
( received using the same buffers as messages in the forward direction.) s
5 575 M
( Therefore they are constrained to be no larger than receive buffers) s
5 564 M
( posted for forward messages.) s
5 542 M
( It is expected that the Upper Layer Protocol consumer establishes an) s
5 531 M
( appropriate payload size limit for backward direction operations,) s
5 520 M
( either by advertising that size limit to its peers, or by convention.) s
5 509 M
( If that is done, backward direction messages do not exceed the size) s
5 498 M
( of receive buffers at either endpoint.) s
5 476 M
( If a sender transmits a backward direction message that is larger) s
5 465 M
( than the receiver is prepared for, the RDMA provider drops the) s
5 454 M
( message and the RDMA connection.) s
5 432 M
(7.3.4. Sending A Backward Direction Call) s
5 410 M
( To form a backward direction RPC-over-RDMA Call message on an RPC-) s
5 399 M
( over-RDMA Version One transport, an ONC RPC service endpoint) s
5 388 M
( constructs an RPC-over-RDMA header containing a fresh RPC XID in the) s
5 377 M
( rdma_xid field.) s
5 355 M
( The rdma_vers field MUST contain the value one. The number of) s
5 344 M
( requested credits is placed in the rdma_credit field.) s
5 322 M
( The rdma_proc field in the RPC-over-RDMA header MUST contain the) s
5 311 M
( value RDMA_MSG. All three chunk lists MUST be empty.) s
5 289 M
( The ONC RPC Call header MUST follow immediately, starting with the) s
5 278 M
( same XID value that is present in the RPC-over-RDMA header. The Call) s
5 267 M
( header's msg_type field MUST contain the value CALL.) s
5 245 M
(7.3.5. Sending A Backward Direction Reply) s
5 223 M
( To form a backward direction RPC-over-RDMA Reply message on an RPC-) s
5 212 M
( over-RDMA Version One transport, an ONC RPC client endpoint) s
5 201 M
( constructs an RPC-over-RDMA header containing a copy of the matching) s
5 190 M
( ONC RPC Call's RPC XID in the rdma_xid field.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 44]) s
_R
S
%%Page: (45) 45
%%BeginPageSetup
_S
76 24 translate
/pagenum 45 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( The rdma_vers field MUST contain the value one. The number of) s
5 674 M
( granted credits is placed in the rdma_credit field.) s
5 652 M
( The rdma_proc field in the RPC-over-RDMA header MUST contain the) s
5 641 M
( value RDMA_MSG. All three chunk lists MUST be empty.) s
5 619 M
( The ONC RPC Reply header MUST follow immediately, starting with the) s
5 608 M
( same XID value that is present in the RPC-over-RDMA header. The) s
5 597 M
( Reply header's msg_type field MUST contain the value REPLY.) s
5 575 M
(7.4. Backward Direction Upper Layer Binding) s
5 553 M
( RPC programs that operate on RPC-over-RDMA Version One only in the) s
5 542 M
( backward direction do not require an Upper Layer Binding) s
5 531 M
( specification. Because RPC-over-RDMA Version One operation in the) s
5 520 M
( backward direction does not allow reduction, there can be no DDP-) s
5 509 M
( eligible data items in such a program. Backward direction operation) s
5 498 M
( occurs on an already-established connection, thus there is no need to) s
5 487 M
( specify RPC bind parameters.) s
5 465 M
(8. Upper Layer Binding Specifications) s
5 443 M
( An Upper Layer Protocol is typically defined independently of any) s
5 432 M
( particular RPC transport. An Upper Layer Binding specification \(ULB\)) s
5 421 M
( provides guidance that helps the Upper Layer Protocol interoperate) s
5 410 M
( correctly and efficiently over a particular transport. For RPC-over-) s
5 399 M
( RDMA Version One, a ULB provides:) s
5 377 M
( o A taxonomy of XDR data items that are eligible for Direct Data) s
5 366 M
( Placement) s
5 344 M
( o A method for determining the maximum size of the reply Payload) s
5 333 M
( stream for all procedures in the Upper Layer Protocol) s
5 311 M
( o An rpcbind port assignment for operation of the RPC Program and) s
5 300 M
( Version on an RPC-over-RDMA transport) s
5 278 M
( Each RPC Program and Version tuple that utilizes RPC-over-RDMA) s
5 267 M
( Version One needs to have an Upper Layer Binding specification.) s
5 256 M
( Requesters MUST NOT send RPC-over-RDMA messages for Upper Layer) s
5 245 M
( Protocols that do not have a Upper Layer Binding. Responders MUST) s
5 234 M
( NOT reply to RPC-over-RDMA messages for Upper Layer Protocols that do) s
5 223 M
( not have a Upper Layer Binding.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 45]) s
_R
S
%%Page: (46) 46
%%BeginPageSetup
_S
76 24 translate
/pagenum 46 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
(8.1. DDP-Eligibility) s
5 663 M
( An Upper Layer Binding designates some XDR data items as eligible for) s
5 652 M
( Direct Data Placement. As an RPC-over-RDMA message is formed, DDP-) s
5 641 M
( eligible data items can be removed from the Payload stream and placed) s
5 630 M
( directly in the receiver's memory \(reduced\).) s
5 608 M
( An XDR data item should be considered for DDP-eligibility if there is) s
5 597 M
( a clear benefit to moving the contents of the item directly from the) s
5 586 M
( sender's memory to the receiver's memory. Criteria for DDP-) s
5 575 M
( eligibility include:) s
5 553 M
( o The XDR data item is frequently sent or received, and its size is) s
5 542 M
( often much larger than typical inline thresholds.) s
5 520 M
( o Transport-level processing of the XDR data item is not needed.) s
5 509 M
( For example, the data item is an opaque byte array, which requires) s
5 498 M
( no XDR encoding and decoding of its content.) s
5 476 M
( o The content of the XDR data item is sensitive to address) s
5 465 M
( alignment. For example, pullup would be required on the receiver) s
5 454 M
( before the content of the item can be used.) s
5 432 M
( o The XDR data item does not contain DDP-eligible data items.) s
5 410 M
( Senders MUST NOT reduce data items that are not DDP-eligible. Such) s
5 399 M
( data items MAY, however, be moved as part of a Position Zero Read) s
5 388 M
( Chunk or a Reply chunk.) s
5 366 M
( The interface by which an Upper Layer implementation indicates the) s
5 355 M
( DDP-eligibility of a data item to the RPC transport is not described) s
5 344 M
( by this specification. The only requirements are that the receiver) s
5 333 M
( can re-assemble the transmitted RPC-over-RDMA message into a valid) s
5 322 M
( XDR stream, and that DDP-eligibility rules specified by the Upper) s
5 311 M
( Layer Binding are respected.) s
5 289 M
( There is no provision to express DDP-eligibility within the XDR) s
5 278 M
( language. The only definitive specification of DDP-eligibility is) s
5 267 M
( the Upper Layer Binding itself.) s
5 245 M
(8.1.1. DDP-Eligibility Violation) s
5 223 M
( A DDP-eligibility violation occurs when a requester forms a Call) s
5 212 M
( message with a non-DDP-eligible data item in a Read chunk. A) s
5 201 M
( violation occurs when a responder forms a Reply message without) s
5 190 M
( reducing a DDP-eligible data item when there is a Write list provided) s
5 179 M
( by the requester.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 46]) s
_R
S
%%Page: (47) 47
%%BeginPageSetup
_S
76 24 translate
/pagenum 47 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( In the first case, a responder MUST NOT process the Call message.) s
5 663 M
( In the second case, as a requester parses a Reply message, it must) s
5 652 M
( assume that the responder has correctly reduced a DDP-eligible result) s
5 641 M
( data item. If the responder has not done so, it is likely that the) s
5 630 M
( requester cannot finish parsing the Payload stream and that an XDR) s
5 619 M
( error would result.) s
5 597 M
( Both types of violations MUST be reported as described in) s
5 586 M
( Section 5.5.2.) s
5 564 M
(8.2. Maximum Reply Size) s
5 542 M
( A requester provides resources for both a Call message and its) s
5 531 M
( matching Reply message. A requester forms the Call message itself,) s
5 520 M
( thus can compute the exact resources needed for it.) s
5 498 M
( A requester must allocate resources for the Reply message \(an RPC-) s
5 487 M
( over-RDMA credit, a Receive buffer, and possibly a Write list and) s
5 476 M
( Reply chunk\) before the responder has formed the actual reply. To) s
5 465 M
( accommodate all possible replies for the procedure in the Call) s
5 454 M
( message, a requester must allocate reply resources based on the) s
5 443 M
( maximum possible size of the expected Reply message.) s
5 421 M
( If there are procedures in the Upper Layer Protocol for which there) s
5 410 M
( is no clear reply size maximum, the Upper Layer Binding needs to) s
5 399 M
( specify a dependable means for determining the maximum.) s
5 377 M
(8.3. Additional Considerations) s
5 355 M
( There may be other details provided in an Upper Layer Binding.) s
5 333 M
( o An Upper Layer Binding may recommend an inline threshold value or) s
5 322 M
( other transport-related parameters for RPC-over-RDMA Version One) s
5 311 M
( connections bearing that Upper Layer Protocol.) s
5 289 M
( o An Upper Layer Protocol may provide a means to communicate these) s
5 278 M
( transport-related parameters between peers. Note that RPC-over-) s
5 267 M
( RDMA Version One does not specify any mechanism for changing any) s
5 256 M
( transport-related parameter after a connection has been) s
5 245 M
( established.) s
5 223 M
( o Multiple Upper Layer Protocols may share a single RPC-over-RDMA) s
5 212 M
( Version One connection when their Upper Layer Bindings allow the) s
5 201 M
( use of RPC-over-RDMA Version One and the rpcbind port assignments) s
5 190 M
( for the Protocols allow connection sharing. In this case, the) s
5 179 M
( same transport parameters \(such as inline threshold\) apply to all) s
5 168 M
( Protocols using that connection.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 47]) s
_R
S
%%Page: (48) 48
%%BeginPageSetup
_S
76 24 translate
/pagenum 48 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( Given the above, Upper Layer Bindings and Upper Layer Protocols must) s
5 674 M
( be designed to interoperate correctly no matter what connection) s
5 663 M
( parameters are in effect on a connection.) s
5 641 M
(8.4. Upper Layer Protocol Extensions) s
5 619 M
( An RPC Program and Version tuple may be extensible. For instance,) s
5 608 M
( there may be a minor versioning scheme that is not reflected in the) s
5 597 M
( RPC version number. Or, the Upper Layer Protocol may allow) s
5 586 M
( additional features to be specified after the original RPC program) s
5 575 M
( specification was ratified.) s
5 553 M
( Upper Layer Bindings are provided for interoperable RPC Programs and) s
5 542 M
( Versions by extending existing Upper Layer Bindings to reflect the) s
5 531 M
( changes made necessary by each addition to the existing XDR.) s
5 509 M
(9. Protocol Extensibility) s
5 487 M
( The RPC-over-RDMA header format is specified using XDR, unlike the) s
5 476 M
( message header format of RPC on TCP. Defining the header using XDR) s
5 465 M
( allows minor issues with the transport protocol to be addressed and) s
5 454 M
( optional features to be introduced by making extensions to the RPC-) s
5 443 M
( over-RDMA header XDR. Such changes can be made without a change to) s
5 432 M
( the protocol version number.) s
5 410 M
( When more invasive changes to the protocol are to be made, a protocol) s
5 399 M
( version number change is required. In either case, any changes to) s
5 388 M
( the RPC-over-RDMA protocol can only be effected by publication of a) s
5 377 M
( Standards Track document with appropriate review by the nfsv4 Working) s
5 366 M
( Group and the IESG.) s
5 344 M
( Although it is possible to make XDR changes which are not limited to) s
5 333 M
( the use of compatible extensions in new RPC-over-RDMA versions, such) s
5 322 M
( changes should only be done when absolutely necessary, as they limit) s
5 311 M
( interoperability with existing implementations. It is appropriate) s
5 300 M
( for the nfsv4 Working Group to consider alternatives carefully before) s
5 289 M
( using this approach.) s
5 267 M
( Unlike the rest of this document, which defines the base of RPC-over-) s
5 256 M
( RDMA Version One, Section 9 \(except for Section 9.4\) applies to all) s
5 245 M
( versions of RPC-over-RDMA. New versions of RPC-over-RDMA may be) s
5 234 M
( published as separate protocols without updating this document, but) s
5 223 M
( any change to the extensibility model defined here requires) s
5 212 M
( publication of a Standards Track document updating this document.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 48]) s
_R
S
%%Page: (49) 49
%%BeginPageSetup
_S
76 24 translate
/pagenum 49 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
(9.1. Changes To RPC-Over-RDMA Header XDR) s
5 663 M
( The first four fields in the RPC-over-RDMA header \(now in struct) s
5 652 M
( rpcrdma_prefix\) must remain aligned at the same fixed offsets for all) s
5 641 M
( versions of the RPC-over-RDMA protocol. The version number must be) s
5 630 M
( in a fixed place in order to enable version mismatch detection. For) s
5 619 M
( version mismatches to be reported in a fashion that all future) s
5 608 M
( version implementations can reliably decode, the rdma_which field) s
5 597 M
( must be in a fixed place, the value of RDMA_ERR_VERS must always) s
5 586 M
( remain the same, and the field placement of the RDMA_ERR_VERS arm of) s
5 575 M
( the rpcrdma1_error union \(now in struct rpcrdma_err_vers\) must always) s
5 564 M
( remain the same.) s
5 542 M
( Given these constraints, one way to extend RPC-over-RDMA is to add) s
5 531 M
( new values to the rdma_proc enumerated type and new components \(arms\)) s
5 520 M
( to the rpcrdma1_body union. New argument and result types may be) s
5 509 M
( introduced for each new procedure defined this way. These extensions) s
5 498 M
( would be specified by new Internet Drafts with appropriate Working) s
5 487 M
( Group and IESG review to ensure continued interoperation with) s
5 476 M
( existing implementations.) s
5 454 M
( XDR extensions may introduce only optional features to an existing) s
5 443 M
( RPC-over-RDMA protocol version. To detect when an optional rdma_proc) s
5 432 M
( value is supported by a receiver, it is desirable to have a specific) s
5 421 M
( value of the rdma_err field, say, RDMA_ERR_PROC, that indicates when) s
5 410 M
( the receiver does not recognize an rdma_proc value.) s
5 388 M
( In RPC-over-RDMA Version One, a receiver can indicate that it does) s
5 377 M
( not recognize an rdma_proc enum value only by returning an RDMA_ERROR) s
5 366 M
( procedure with the rdma_err field set to RDMA_ERR_BADHEADER \(see) s
5 355 M
( Section 5.5.2\). This is indistinguishable from a situation where the) s
5 344 M
( receiver does indeed support the procedure, but the XDR is malformed.) s
5 322 M
( To resolve this problem, an RPC-over-RDMA Version One sender uses the) s
5 311 M
( following convention. If the first time the sender uses an optional) s
5 300 M
( rdma_proc value the receiver returns an RDMA_ERROR procedure with) s
5 289 M
( RDMA_ERR_BADHEADER in the rdma_err field, the sender simply marks) s
5 278 M
( that feature as unsupported and does not send it again on the current) s
5 267 M
( connection instance. Subsequent to an initial successful result,) s
5 256 M
( receiving RDMA_ERR_BADHEADER retains its more relaxed meaning of) s
5 245 M
( "generic XDR parsing error.") s
5 223 M
( To ensure backwards compatibility when such an extension mechanism is) s
5 212 M
( in place, the value of RDMA_ERR_BADHEADER must remain the same for) s
5 201 M
( all versions of the RPC-over-RDMA protocol.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 49]) s
_R
S
%%Page: (50) 50
%%BeginPageSetup
_S
76 24 translate
/pagenum 50 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( Most changes to the RPC-over-RDMA XDR will take the form of a) s
5 674 M
( compatible extension to the existing XDR. Changes which do not) s
5 663 M
( update the version number \(see Section 9.3\) must take this form.) s
5 641 M
( For an XDR description B to be a compatible extension of an XDR) s
5 630 M
( description A, the following must be the case:) s
5 608 M
( o All input recognized as description valid by A must be recognized) s
5 597 M
( as valid by description B) s
5 575 M
( o Any input recognized as valid by both descriptions must be) s
5 564 M
( interpreted as having the same structure according to both) s
5 553 M
( descriptions) s
5 531 M
( o Any input recognized as valid by description B but not by) s
5 520 M
( description A can be recognizable as part of a supported./unknown) s
5 509 M
( extension using description A) s
5 487 M
( The following changes can be made compatibly:) s
5 465 M
( o Addition of a new message header type and associated header body) s
5 443 M
( o Addition of new enum values and associated arms to unions that do) s
5 432 M
( not include a default case) s
5 410 M
( o Addition of previously undefined flag bits to flag words that are) s
5 399 M
( included in existing header bodies) s
5 377 M
( Each such addition is referred to as a "protocol element." A set of) s
5 366 M
( protocol elements defined together such that all must be supported or) s
5 355 M
( not supported by a receiver is called a "feature.") s
5 333 M
( Because of the simplicity of the existing protocol and deficiencies) s
5 322 M
( in the existing error reporting structure, some of the above) s
5 311 M
( techiques are not realizable within RPC-over-RDMA Version One. For a) s
5 300 M
( discussion of protocol extension practices within RPC-over-RDMA) s
5 289 M
( Version One, including XDR extension, see Section 9.4.) s
5 267 M
(9.2. Feature Statuses With RPC-Over-RDMA Versions) s
5 245 M
( Within a given RPC-over-RDMA version, every known feature is either) s
5 234 M
( OPTIONAL, REQUIRED, or "not allowed".) s
5 212 M
( o REQUIRED features MUST be supported by all receivers. Senders can) s
5 201 M
( depend on them being supported.) s
5 179 M
( o OPTIONAL features MAY be supported by particular receivers.) s
5 168 M
( Senders need to be prepared for the absence of support.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 50]) s
_R
S
%%Page: (51) 51
%%BeginPageSetup
_S
76 24 translate
/pagenum 51 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( o "Not allowed" features are typically those that were formally) s
5 674 M
( OPTIONAL or REQUIRED, but for which support has been removed.) s
5 652 M
( All features defined in this document are REQUIRED in RPC-over-RDMA) s
5 641 M
( Version One. OPTIONAL features may be added to Version One as) s
5 630 M
( specified in Section 9.4.) s
5 608 M
( The terms "OPTIONAL" and "REQUIRED" are used as specified in) s
5 597 M
( [RFC2119] as indicated in Section 1.1. These status values are) s
5 586 M
( assigned by those writing additional specifications \(e.g., new RPC-) s
5 575 M
( over-RDMA versions or extensions to existing RPC-over-RDMA versions\).) s
5 564 M
( Their choice in this regard is their guidance to implementers. As) s
5 553 M
( used in this document, these terms are only directed to implementers) s
5 542 M
( of RPC-over-RDMA Version One.) s
5 520 M
( The status of features may change between RPC-over-RDMA protocol) s
5 509 M
( versions.) s
5 487 M
(9.3. RPC-Over-RDMA Version Numbering) s
5 465 M
( RPC-over-RDMA version numbering enables both endpoints to agree on a) s
5 454 M
( set of interoperable behaviors and determine which OPTIONAL features) s
5 443 M
( are available.) s
5 421 M
( An expected pattern of protocol development is to introduce OPTIONAL) s
5 410 M
( features within a given version using XDR extension. Such features) s
5 399 M
( often need a significant period of optional general use to ensure) s
5 388 M
( they are capable of being implemented broadly. This is especially) s
5 377 M
( true for infrastructural features that others will build upon. When) s
5 366 M
( it is appropriate for OPTIONAL features to become REQUIRED, that) s
5 355 M
( would be an occasion to create a new RPC-over-RDMA protocol version.) s
5 333 M
( The value of the RPC-over-RDMA header's version field has to be) s
5 322 M
( updated when the protocol is altered in a way that prevents) s
5 311 M
( interoperability with current implementations. A version change is) s
5 300 M
( needed whenever:) s
5 278 M
( o The RPC-over-RDMA header XDR definition is changed to add a) s
5 267 M
( REQUIRED protocol element, or an existing OPTIONAL feature is made) s
5 256 M
( REQUIRED) s
5 234 M
( o A REQUIRED feature is made OPTIONAL) s
5 212 M
( o A REQUIRED or OPTIONAL feature is converted to be "not allowed") s
5 190 M
( o An XDR change is made that is not a compatible extension as) s
5 179 M
( defined in Section 9.1) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 51]) s
_R
S
%%Page: (52) 52
%%BeginPageSetup
_S
76 24 translate
/pagenum 52 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( o The use of a previously not used abstract RDMA operation is) s
5 674 M
( specified as REQUIRED) s
5 652 M
( o The use of an existing REQUIRED abstract RDMA operation is removed) s
5 630 M
( When a version number change is to be made, the nfsv4 Working Group) s
5 619 M
( creates a Standards Track document that does one of the following:) s
5 597 M
( 1. Documents the whole protocol as amended) s
5 575 M
( 2. Documents changes relative to the previous minor version) s
5 553 M
( 3. Documents extensions made since the previous minor versions by) s
5 542 M
( normatively referencing the documents defining those extensions) s
5 520 M
( 4. Documents all REQUIRED functionality, and includes OPTIONAL) s
5 509 M
( features by normatively referencing the documents defining those) s
5 498 M
( extensions) s
5 476 M
( The Working Group retains all these options, but the last is) s
5 465 M
( typically preferred. When an XDR change that is not a compatible) s
5 454 M
( extension is made, the first is most desirable. In any case, if) s
5 443 M
( there are features whose status has been changed to "not allowed",) s
5 432 M
( the document needs to explain that change and how it is intended that) s
5 421 M
( existing implementations address the feature removal.) s
5 399 M
(9.4. RPC-Over-RDMA Version One Extension Practices) s
5 377 M
( This subsection applies primarily to RPC-over-RDMA Version One but) s
5 366 M
( remains in effect unless modified by documents defining future RPC-) s
5 355 M
( over-RDMA versions. Such documents need not update this document.) s
5 333 M
(9.4.1. Documentation Requirements) s
5 311 M
( RPC-over-RDMA Version One may be extended by defining a new message) s
5 300 M
( header type and XDR description of the corresponding header body.) s
5 278 M
( A set of such new protocol elements may be introduced by a Standards) s
5 267 M
( Track document and are together considered an OPTIONAL feature.) s
5 256 M
( nfsv4 Working Group and IESG review, together with appropriate) s
5 245 M
( testing of prototype implementations, should ensure continued) s
5 234 M
( interoperation with existing implementations.) s
5 212 M
( Documents describing extensions to RPC-over-RDMA Version One should) s
5 201 M
( contain:) s
5 179 M
( o An explanation of the purpose and use of each new protocol element) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 52]) s
_R
S
%%Page: (53) 53
%%BeginPageSetup
_S
76 24 translate
/pagenum 53 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( o An XDR description and a script to extract it) s
5 663 M
( o A receiver response that a sender can use to determine that) s
5 652 M
( support is in fact present) s
5 630 M
( o A description of interactions with existing features \(e.g., any) s
5 619 M
( requirement that another OPTIONAL or REQUIRED feature needs to be) s
5 608 M
( present and supported for the new feature to work\)) s
5 586 M
( Implementers concatenate the XDR description of the new feature with) s
5 575 M
( the XDR description of the base protocol, extracted from this) s
5 564 M
( document, to produce a combined XDR description for the RPC-over-RDMA) s
5 553 M
( Version One protocol with the specified extension.) s
5 531 M
(9.4.2. Detecting Support For Message Header Types) s
5 509 M
( A sender determines whether a receiver supports an OPTIONAL message) s
5 498 M
( header type by issuing a simple test request using that message) s
5 487 M
( header type. The receiver sends an affirmative response that) s
5 476 M
( indicates the message header type is supported. The response message) s
5 465 M
( header type may itself be an extension. The sender ties together the) s
5 454 M
( message and response using the rdma_xid field.) s
5 432 M
( The receiver indicates that it does not recognize a particular) s
5 421 M
( rdma_which value by returning an RDMA_ERROR message type with the) s
5 410 M
( rdma_err field set to RDMA_ERR_BADHEADER and with the rdma_xid field) s
5 399 M
( set to a value that matches the test message.) s
5 377 M
( This is indistinguishable from a situation where the receiver does) s
5 366 M
( support the procedure but the test message is malformed. However, if) s
5 355 M
( the sender always tests for receiver support using a simple instance) s
5 344 M
( of the message header type to be tested, such an error at this point) s
5 333 M
( indicates the sender and receiver have no prospect of using the new) s
5 322 M
( protocol element interoperably. A lack of support for this feature) s
5 311 M
( can be reasonably assumed.) s
5 289 M
( A sender should issue OPTIONAL message header types one-at-a-time) s
5 278 M
( until it receives indication of the receiver's support status of that) s
5 267 M
( message header type.) s
5 245 M
(10. Security Considerations) s
5 223 M
(10.1. Memory Protection) s
5 201 M
( A primary consideration is the protection of the integrity and) s
5 190 M
( privacy of local memory by an RPC-over-RDMA transport. The use of) s
5 179 M
( RPC-over-RDMA MUST NOT introduce any vulnerabilities to system memory) s
5 168 M
( contents, nor to memory owned by user processes.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 53]) s
_R
S
%%Page: (54) 54
%%BeginPageSetup
_S
76 24 translate
/pagenum 54 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( It is REQUIRED that any RDMA provider used for RPC transport be) s
5 674 M
( conformant to the requirements of [RFC5042] in order to satisfy these) s
5 663 M
( protections. These protections are provided by the RDMA layer) s
5 652 M
( specifications, and in particular, their security models.) s
5 630 M
(10.1.1. Protection Domains) s
5 608 M
( The use of Protection Domains to limit the exposure of memory) s
5 597 M
( segments to a single connection is critical. Any attempt by an) s
5 586 M
( endpoint not participating in that connection to re-use memory) s
5 575 M
( handles needs to result in immediate failure of that connection.) s
5 564 M
( Because Upper Layer Protocol security mechanisms rely on this aspect) s
5 553 M
( of Reliable Connection behavior, strong authentication of remote) s
5 542 M
( endpoints is recommended.) s
5 520 M
(10.1.2. Handle Predictability) s
5 498 M
( Unpredictable memory handles should be used for any operation) s
5 487 M
( requiring advertised memory segments. Advertising a continuously) s
5 476 M
( registered memory region allows a remote host to read or write to) s
5 465 M
( that region even when an RPC involving that memory is not under way.) s
5 454 M
( Therefore implementations should avoid advertising persistently) s
5 443 M
( registered memory.) s
5 421 M
(10.1.3. Memory Fencing) s
5 399 M
( Advertised memory segments should be invalidated as soon as related) s
5 388 M
( RPC operations are complete. Invalidation and DMA unmapping of) s
5 377 M
( segments should be complete before the Upper Layer is allowed to) s
5 366 M
( continue execution and use or alter the contents of a memory region.) s
5 344 M
(10.2. RPC Message Security) s
5 322 M
( ONC RPC provides cryptographic security via the RPCSEC_GSS framework) s
5 311 M
( [RFC2203]. RPCSEC_GSS implements message authentication, per-message) s
5 300 M
( integrity checking, and per-message confidentiality. However,) s
5 289 M
( integrity and privacy services require significant movement of data) s
5 278 M
( on each endpoint host. Some performance benefits enabled by RDMA) s
5 267 M
( transports can be lost. Note that some performance loss is expected) s
5 256 M
( when RPCSEC_GSS integrity or privacy is in use on any RPC transport.) s
5 234 M
(10.2.1. RPC-Over-RDMA Link-Level Protection) s
5 212 M
( Link-level protection is a more appropriate security mechanism for) s
5 201 M
( RDMA transports. Certain configurations of IPsec can be co-located) s
5 190 M
( in RDMA hardware, for example, without any change to RDMA consumers) s
5 179 M
( or loss of data movement efficiency.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 54]) s
_R
S
%%Page: (55) 55
%%BeginPageSetup
_S
76 24 translate
/pagenum 55 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( The use of link-level protection MAY be negotiated through the use of) s
5 674 M
( the RPCSEC_GSS security flavor defined in [RFC5403] in conjunction) s
5 663 M
( with the Channel Binding mechanism [RFC5056] and IPsec Channel) s
5 652 M
( Connection Latching [RFC5660]. Use of such mechanisms is REQUIRED) s
5 641 M
( where integrity and/or privacy is desired and where efficiency is) s
5 630 M
( required.) s
5 608 M
(10.2.2. RPCSEC_GSS On RPC-Over-RDMA Transports) s
5 586 M
( RPCSEC_GSS [RFC5403] extends the ONC RPC protocol [RFC5531] without) s
5 575 M
( changing the format of RPC messages. By observing the conventions) s
5 564 M
( described in this section, an RPC-over-RDMA implementation can) s
5 553 M
( support RPCSEC_GSS in a way that interoperates successfully with) s
5 542 M
( other implementations.) s
5 520 M
( As part of the ONC RPC protocol, protocol elements of RPCSEC_GSS that) s
5 509 M
( appear in the Payload stream of an RPC-over-RDMA message \(such as) s
5 498 M
( control messages exchanged as part of establishing or destroying a) s
5 487 M
( security context, or data items that are part of RPCSEC_GSS) s
5 476 M
( authentication material\) MUST NOT be reduced.) s
5 454 M
(10.2.2.1. RPCSEC_GSS Context Negotiation) s
5 432 M
( Some NFS client implementations use a separate connection to) s
5 421 M
( establish a GSS context for NFS operation. These clients use TCP and) s
5 410 M
( the standard NFS port \(2049\) for context establishment, but there is) s
5 399 M
( no guarantee that an NFS/RDMA server provides a TCP-based NFS server) s
5 388 M
( on port 2049.) s
5 366 M
(10.2.2.2. RPC-Over-RDMA With RPCSEC_GSS Authentication) s
5 344 M
( The RPCSEC_GSS authentication service has no impact on the DDP-) s
5 333 M
( eligibity of data items in an Upper Layer Protocol.) s
5 311 M
( However, RPCSEC_GSS authentication material appearing in an RPC) s
5 300 M
( message header is often larger than material associated with, say,) s
5 289 M
( the AUTH_SYS security flavor. In particular, when an RPCSEC_GSS) s
5 278 M
( pseudoflavor is in use, a requester needs to accommodate a larger RPC) s
5 267 M
( credential when marshaling Call messages, and to provide for a) s
5 256 M
( maximum size RPCSEC_GSS verifier when allocating reply buffers and) s
5 245 M
( Reply chunks.) s
5 223 M
( RPC messages, and thus Payload streams, are made larger as a result.) s
5 212 M
( Upper Layer Protocol operations that fit in a Short Message when a) s
5 201 M
( simpler form of authentication is in use might need to be reduced or) s
5 190 M
( conveyed via a Long Message when RPCSEC_GSS authentication is in use.) s
5 179 M
( This can impact efficiency when RPCSEC_GSS authentication is use.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 55]) s
_R
S
%%Page: (56) 56
%%BeginPageSetup
_S
76 24 translate
/pagenum 56 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( Because average RPC message size is larger when RPCSEC_GSS) s
5 674 M
( authentication is in use, it is more likely that a requester will) s
5 663 M
( provide both a Read list and a Reply chunk in the same RPC-over-RDMA) s
5 652 M
( header to convey a Long call and provision a receptacle for a Long) s
5 641 M
( reply.) s
5 619 M
(10.2.2.3. RPC-Over-RDMA With RPCSEC_GSS Integrity Or Privacy) s
5 597 M
( The RPCSEC_GSS integrity service enables endpoints to detect) s
5 586 M
( modification of RPC messages in flight. The RPCSEC_GSS privacy) s
5 575 M
( service prevents all but the intended recipient from viewing the) s
5 564 M
( cleartext content of RPC messages. RPCSEC_GSS integrity and privacy) s
5 553 M
( are end-to-end; that is, they protect RPC arguments and results from) s
5 542 M
( application to server endpoint, and back.) s
5 520 M
( The RPCSEC_GSS integrity and encryption services operate on whole RPC) s
5 509 M
( messages after they have been XDR encoded for transmit, and before) s
5 498 M
( they have been XDR decoded after receipt. Both the sender and the) s
5 487 M
( receiver endpoints use intermediate buffers to prevent exposure of) s
5 476 M
( encrypted data or unverified cleartext data to RPC consumers. After) s
5 465 M
( verification, encryption, and message wrapping has been performed,) s
5 454 M
( the transport layer can use RDMA data transfer between these) s
5 443 M
( intermediate buffers.) s
5 421 M
( The process of reducing a DDP-eligible data item removes the data) s
5 410 M
( item and its XDR padding from the encoded XDR stream. XDR padding of) s
5 399 M
( a reduced data item is not transferred in an RPC-over-RDMA message.) s
5 388 M
( After reduction, the Payload stream contains fewer octets then the) s
5 377 M
( whole XDR stream did beforehand. XDR padding octets are often zero) s
5 366 M
( bytes, but they don't have to be. Thus reducing DDP-eligible items) s
5 355 M
( affects the result of message integrity verification or encryption.) s
5 333 M
( Therefore a sender MUST NOT reduce a Payload stream when RPCSEC_GSS) s
5 322 M
( integrity or encryption services are in use. Effectively, no data) s
5 311 M
( item is DDP-eligible in this situation, and Chunked Messages cannot) s
5 300 M
( be used. In this mode, an RPC-over-RDMA transport operates in the) s
5 289 M
( same manner as a transport that does not support direct data) s
5 278 M
( placement.) s
5 256 M
( When RPCSEC_GSS integrity or privacy is in use, a requester provides) s
5 245 M
( both a Read list and a Reply chunk in the same RPC-over-RDMA header) s
5 234 M
( to convey a Long call and provision a receptacle for a Long reply.) s
5 212 M
(10.2.2.4. RPC-Over-RDMA Header Exposure) s
5 190 M
( Like the base fields in an ONC RPC message \(XID, call direction, and) s
5 179 M
( so on\), the contents of an RPC-over-RDMA message's Transport stream) s
5 168 M
( are not protected by RPCSEC_GSS. This exposes XIDs, connection) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 56]) s
_R
S
%%Page: (57) 57
%%BeginPageSetup
_S
76 24 translate
/pagenum 57 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( credit limits, and chunk lists \(but not the content of the data items) s
5 674 M
( they refer to\) to malicious behavior, which could redirect data that) s
5 663 M
( is transferred by the RPC-over-RDMA message, result in spurious) s
5 652 M
( retransmits, or trigger connection loss.) s
5 630 M
( Encryption at the link layer, as described in Section 10.2.1,) s
5 619 M
( protects the content of the Transport stream.) s
5 597 M
(11. IANA Considerations) s
5 575 M
( Three assignments are specified by this document. These are) s
5 564 M
( unchanged from [RFC5666]:) s
5 542 M
( o A set of RPC "netids" for resolving RPC-over-RDMA services) s
5 520 M
( o Optional service port assignments for Upper Layer Bindings) s
5 498 M
( o An RPC program number assignment for the configuration protocol) s
5 476 M
( These assignments have been established, as below.) s
5 454 M
( The new RPC transport has been assigned an RPC "netid", which is an) s
5 443 M
( rpcbind [RFC1833] string used to describe the underlying protocol in) s
5 432 M
( order for RPC to select the appropriate transport framing, as well as) s
5 421 M
( the format of the service addresses and ports.) s
5 399 M
( The following "Netid" registry strings are defined for this purpose:) s
5 366 M
( NC_RDMA "rdma") s
5 355 M
( NC_RDMA6 "rdma6") s
5 322 M
( These netids MAY be used for any RDMA network satisfying the) s
5 311 M
( requirements of Section 3.2.2, and able to identify service endpoints) s
5 300 M
( using IP port addressing, possibly through use of a translation) s
5 289 M
( service as described above in Section 6. The "rdma" netid is to be) s
5 278 M
( used when IPv4 addressing is employed by the underlying transport,) s
5 267 M
( and "rdma6" for IPv6 addressing.) s
5 245 M
( The netid assignment policy and registry are defined in [RFC5665].) s
5 223 M
( As a new RPC transport, this protocol has no effect on RPC Program) s
5 212 M
( numbers or existing registered port numbers. However, new port) s
5 201 M
( numbers MAY be registered for use by RPC-over-RDMA-enabled services,) s
5 190 M
( as appropriate to the new networks over which the services will) s
5 179 M
( operate.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 57]) s
_R
S
%%Page: (58) 58
%%BeginPageSetup
_S
76 24 translate
/pagenum 58 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( For example, the NFS/RDMA service defined in [RFC5667] has been) s
5 674 M
( assigned the port 20049, in the IANA registry:) s
5 641 M
( nfsrdma 20049/tcp Network File System \(NFS\) over RDMA) s
5 630 M
( nfsrdma 20049/udp Network File System \(NFS\) over RDMA) s
5 619 M
( nfsrdma 20049/sctp Network File System \(NFS\) over RDMA) s
5 586 M
( The RPC program number assignment policy and registry are defined in) s
5 575 M
( [RFC5531].) s
5 553 M
(12. Acknowledgments) s
5 531 M
( The editor gratefully acknowledges the work of Brent Callaghan and) s
5 520 M
( Tom Talpey on the original RPC-over-RDMA Version One specification) s
5 509 M
( [RFC5666].) s
5 487 M
( Dave Noveck provided excellent review, constructive suggestions, and) s
5 476 M
( consistent navigational guidance throughout the process of drafting) s
5 465 M
( this document. Dave also contributed much of the organization and) s
5 454 M
( content of Section 9 and helped the authors understand the) s
5 443 M
( complexities of XDR extensibility.) s
5 421 M
( The comments and contributions of Karen Deitke, Dai Ngo, Chunli) s
5 410 M
( Zhang, Dominique Martinet, and Mahesh Siddheshwar are accepted with) s
5 399 M
( great thanks. The editor also wishes to thank Bill Baker for his) s
5 388 M
( support of this work.) s
5 366 M
( The extract.sh shell script and formatting conventions were first) s
5 355 M
( described by the authors of the NFSv4.1 XDR specification [RFC5662].) s
5 333 M
( Special thanks go to nfsv4 Working Group Chair Spencer Shepler and) s
5 322 M
( nfsv4 Working Group Secretary Thomas Haynes for their support.) s
5 300 M
(13. References) s
5 278 M
(13.1. Normative References) s
5 256 M
( [RFC1833] Srinivasan, R., "Binding Protocols for ONC RPC Version 2",) s
5 245 M
( RFC 1833, DOI 10.17487/RFC1833, August 1995,) s
5 234 M
( <http://www.rfc-editor.org/info/rfc1833>.) s
5 212 M
( [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate) s
5 201 M
( Requirement Levels", BCP 14, RFC 2119, DOI 10.17487/) s
5 190 M
( RFC2119, March 1997,) s
5 179 M
( <http://www.rfc-editor.org/info/rfc2119>.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 58]) s
_R
S
%%Page: (59) 59
%%BeginPageSetup
_S
76 24 translate
/pagenum 59 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( [RFC2203] Eisler, M., Chiu, A., and L. Ling, "RPCSEC_GSS Protocol) s
5 674 M
( Specification", RFC 2203, DOI 10.17487/RFC2203, September) s
5 663 M
( 1997, <http://www.rfc-editor.org/info/rfc2203>.) s
5 641 M
( [RFC4506] Eisler, M., Ed., "XDR: External Data Representation) s
5 630 M
( Standard", STD 67, RFC 4506, DOI 10.17487/RFC4506, May) s
5 619 M
( 2006, <http://www.rfc-editor.org/info/rfc4506>.) s
5 597 M
( [RFC5042] Pinkerton, J. and E. Deleganes, "Direct Data Placement) s
5 586 M
( Protocol \(DDP\) / Remote Direct Memory Access Protocol) s
5 575 M
( \(RDMAP\) Security", RFC 5042, DOI 10.17487/RFC5042, October) s
5 564 M
( 2007, <http://www.rfc-editor.org/info/rfc5042>.) s
5 542 M
( [RFC5056] Williams, N., "On the Use of Channel Bindings to Secure) s
5 531 M
( Channels", RFC 5056, DOI 10.17487/RFC5056, November 2007,) s
5 520 M
( <http://www.rfc-editor.org/info/rfc5056>.) s
5 498 M
( [RFC5403] Eisler, M., "RPCSEC_GSS Version 2", RFC 5403, DOI) s
5 487 M
( 10.17487/RFC5403, February 2009,) s
5 476 M
( <http://www.rfc-editor.org/info/rfc5403>.) s
5 454 M
( [RFC5531] Thurlow, R., "RPC: Remote Procedure Call Protocol) s
5 443 M
( Specification Version 2", RFC 5531, DOI 10.17487/RFC5531,) s
5 432 M
( May 2009, <http://www.rfc-editor.org/info/rfc5531>.) s
5 410 M
( [RFC5660] Williams, N., "IPsec Channels: Connection Latching", RFC) s
5 399 M
( 5660, DOI 10.17487/RFC5660, October 2009,) s
5 388 M
( <http://www.rfc-editor.org/info/rfc5660>.) s
5 366 M
( [RFC5665] Eisler, M., "IANA Considerations for Remote Procedure Call) s
5 355 M
( \(RPC\) Network Identifiers and Universal Address Formats",) s
5 344 M
( RFC 5665, DOI 10.17487/RFC5665, January 2010,) s
5 333 M
( <http://www.rfc-editor.org/info/rfc5665>.) s
5 311 M
(13.2. Informative References) s
5 289 M
( [IB] InfiniBand Trade Association, "InfiniBand Architecture) s
5 278 M
( Specifications", <http://www.infinibandta.org>.) s
5 256 M
( [IBPORT] InfiniBand Trade Association, "IP Addressing Annex",) s
5 245 M
( <http://www.infinibandta.org>.) s
5 223 M
( [RFC0768] Postel, J., "User Datagram Protocol", STD 6, RFC 768, DOI) s
5 212 M
( 10.17487/RFC0768, August 1980,) s
5 201 M
( <http://www.rfc-editor.org/info/rfc768>.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 59]) s
_R
S
%%Page: (60) 60
%%BeginPageSetup
_S
76 24 translate
/pagenum 60 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( [RFC0793] Postel, J., "Transmission Control Protocol", STD 7, RFC) s
5 674 M
( 793, DOI 10.17487/RFC0793, September 1981,) s
5 663 M
( <http://www.rfc-editor.org/info/rfc793>.) s
5 641 M
( [RFC1094] Nowicki, B., "NFS: Network File System Protocol) s
5 630 M
( specification", RFC 1094, DOI 10.17487/RFC1094, March) s
5 619 M
( 1989, <http://www.rfc-editor.org/info/rfc1094>.) s
5 597 M
( [RFC1813] Callaghan, B., Pawlowski, B., and P. Staubach, "NFS) s
5 586 M
( Version 3 Protocol Specification", RFC 1813, DOI 10.17487/) s
5 575 M
( RFC1813, June 1995,) s
5 564 M
( <http://www.rfc-editor.org/info/rfc1813>.) s
5 542 M
( [RFC5040] Recio, R., Metzler, B., Culley, P., Hilland, J., and D.) s
5 531 M
( Garcia, "A Remote Direct Memory Access Protocol) s
5 520 M
( Specification", RFC 5040, DOI 10.17487/RFC5040, October) s
5 509 M
( 2007, <http://www.rfc-editor.org/info/rfc5040>.) s
5 487 M
( [RFC5041] Shah, H., Pinkerton, J., Recio, R., and P. Culley, "Direct) s
5 476 M
( Data Placement over Reliable Transports", RFC 5041, DOI) s
5 465 M
( 10.17487/RFC5041, October 2007,) s
5 454 M
( <http://www.rfc-editor.org/info/rfc5041>.) s
5 432 M
( [RFC5532] Talpey, T. and C. Juszczak, "Network File System \(NFS\)) s
5 421 M
( Remote Direct Memory Access \(RDMA\) Problem Statement", RFC) s
5 410 M
( 5532, DOI 10.17487/RFC5532, May 2009,) s
5 399 M
( <http://www.rfc-editor.org/info/rfc5532>.) s
5 377 M
( [RFC5661] Shepler, S., Ed., Eisler, M., Ed., and D. Noveck, Ed.,) s
5 366 M
( "Network File System \(NFS\) Version 4 Minor Version 1) s
5 355 M
( Protocol", RFC 5661, DOI 10.17487/RFC5661, January 2010,) s
5 344 M
( <http://www.rfc-editor.org/info/rfc5661>.) s
5 322 M
( [RFC5662] Shepler, S., Ed., Eisler, M., Ed., and D. Noveck, Ed.,) s
5 311 M
( "Network File System \(NFS\) Version 4 Minor Version 1) s
5 300 M
( External Data Representation Standard \(XDR\) Description",) s
5 289 M
( RFC 5662, DOI 10.17487/RFC5662, January 2010,) s
5 278 M
( <http://www.rfc-editor.org/info/rfc5662>.) s
5 256 M
( [RFC5666] Talpey, T. and B. Callaghan, "Remote Direct Memory Access) s
5 245 M
( Transport for Remote Procedure Call", RFC 5666, DOI) s
5 234 M
( 10.17487/RFC5666, January 2010,) s
5 223 M
( <http://www.rfc-editor.org/info/rfc5666>.) s
5 201 M
( [RFC5667] Talpey, T. and B. Callaghan, "Network File System \(NFS\)) s
5 190 M
( Direct Data Placement", RFC 5667, DOI 10.17487/RFC5667,) s
5 179 M
( January 2010, <http://www.rfc-editor.org/info/rfc5667>.) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 60]) s
_R
S
%%Page: (61) 61
%%BeginPageSetup
_S
76 24 translate
/pagenum 61 def
/fname (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/fdir (.) def
/ftail (draft-ietf-nfsv4-rfc5666bis-04.txt) def
/user_header_p false def
/user_footer_p false def
%%EndPageSetup
5 718 M
(Internet-Draft RDMA Transport for RPC March 2016) s
5 685 M
( [RFC7530] Haynes, T., Ed. and D. Noveck, Ed., "Network File System) s
5 674 M
( \(NFS\) Version 4 Protocol", RFC 7530, DOI 10.17487/RFC7530,) s
5 663 M
( March 2015, <http://www.rfc-editor.org/info/rfc7530>.) s
5 641 M
(Authors' Addresses) s
5 619 M
( Charles Lever \(editor\)) s
5 608 M
( Oracle Corporation) s
5 597 M
( 1015 Granger Avenue) s
5 586 M
( Ann Arbor, MI 48104) s
5 575 M
( USA) s
5 553 M
( Phone: +1 734 274 2396) s
5 542 M
( Email: chuck.lever@oracle.com) s
5 509 M
( William Allen Simpson) s
5 498 M
( DayDreamer) s
5 487 M
( 1384 Fontaine) s
5 476 M
( Madison Heights, MI 48071) s
5 465 M
( USA) s
5 443 M
( Email: william.allen.simpson@gmail.com) s
5 410 M
( Tom Talpey) s
5 399 M
( Microsoft Corp.) s
5 388 M
( One Microsoft Way) s
5 377 M
( Redmond, WA 98052) s
5 366 M
( USA) s
5 344 M
( Phone: +1 425 704-9945) s
5 333 M
( Email: ttalpey@microsoft.com) s
5 124 M
(Lever, et al. Expires September 5, 2016 [Page 61]) s
_R
S
%%Trailer
%%Pages: 61
%%DocumentNeededResources: font Courier-Bold Courier
%%EOF
| PAFTECH AB 2003-2026 | 2026-04-22 17:05:41 |