<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<!--[if !mso]><style>v\:* {behavior:url(#default#VML);}
o\:* {behavior:url(#default#VML);}
w\:* {behavior:url(#default#VML);}
.shape {behavior:url(#default#VML);}
</style><![endif]--><style><!--
/* Font Definitions */
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:"Lucida Grande";}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0cm;
margin-bottom:.0001pt;
font-size:11.0pt;
font-family:"Calibri",sans-serif;
mso-fareast-language:EN-US;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:#0563C1;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-priority:99;
color:#954F72;
text-decoration:underline;}
span.E-postmall17
{mso-style-type:personal-compose;
font-family:"Calibri",sans-serif;
color:windowtext;}
.MsoChpDefault
{mso-style-type:export-only;
font-family:"Calibri",sans-serif;
mso-fareast-language:EN-US;}
@page WordSection1
{size:612.0pt 792.0pt;
margin:70.85pt 70.85pt 70.85pt 70.85pt;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:218592919;
mso-list-template-ids:0;}
@list l0:level1
{mso-level-text:"%1\)";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:18.0pt;
text-indent:-18.0pt;}
@list l0:level2
{mso-level-number-format:alpha-lower;
mso-level-text:"%2\)";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:36.0pt;
text-indent:-18.0pt;}
@list l0:level3
{mso-level-number-format:roman-lower;
mso-level-text:"%3\)";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:54.0pt;
text-indent:-18.0pt;}
@list l0:level4
{mso-level-text:"\(%4\)";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:72.0pt;
text-indent:-18.0pt;}
@list l0:level5
{mso-level-number-format:alpha-lower;
mso-level-text:"\(%5\)";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:90.0pt;
text-indent:-18.0pt;}
@list l0:level6
{mso-level-number-format:roman-lower;
mso-level-text:"\(%6\)";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:108.0pt;
text-indent:-18.0pt;}
@list l0:level7
{mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:126.0pt;
text-indent:-18.0pt;}
@list l0:level8
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:144.0pt;
text-indent:-18.0pt;}
@list l0:level9
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:162.0pt;
text-indent:-18.0pt;}
@list l0:level1 lfo1
{mso-level-numbering:continue;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level2 lfo1
{mso-level-numbering:continue;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level3 lfo1
{mso-level-numbering:continue;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level4 lfo1
{mso-level-numbering:continue;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level5 lfo1
{mso-level-numbering:continue;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level6 lfo1
{mso-level-numbering:continue;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level7 lfo1
{mso-level-numbering:continue;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level8 lfo1
{mso-level-numbering:continue;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level9 lfo1
{mso-level-numbering:continue;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level1 lfo2
{mso-level-number-format:roman-upper;
mso-level-numbering:continue;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level2 lfo2
{mso-level-number-format:alpha-upper;
mso-level-numbering:continue;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level3 lfo2
{mso-level-number-format:arabic;
mso-level-numbering:continue;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level4 lfo2
{mso-level-number-format:alpha-lower;
mso-level-numbering:continue;
mso-level-text:"%4\)";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level5 lfo2
{mso-level-number-format:arabic;
mso-level-numbering:continue;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level6 lfo2
{mso-level-number-format:alpha-lower;
mso-level-numbering:continue;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level7 lfo2
{mso-level-number-format:roman-lower;
mso-level-numbering:continue;
mso-level-text:"\(%7\)";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level8 lfo2
{mso-level-numbering:continue;
mso-level-text:"\(%8\)";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level9 lfo2
{mso-level-numbering:continue;
mso-level-text:"\(%9\)";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level1 lfo3
{mso-level-numbering:continue;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level2 lfo3
{mso-level-number-format:arabic;
mso-level-numbering:continue;
mso-level-text:"%1\.%2\.";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level3 lfo3
{mso-level-number-format:arabic;
mso-level-numbering:continue;
mso-level-text:"%1\.%2\.%3\.";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level4 lfo3
{mso-level-numbering:continue;
mso-level-text:"%1\.%2\.%3\.%4\.";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level5 lfo3
{mso-level-number-format:arabic;
mso-level-numbering:continue;
mso-level-text:"%1\.%2\.%3\.%4\.%5\.";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level6 lfo3
{mso-level-number-format:arabic;
mso-level-numbering:continue;
mso-level-text:"%1\.%2\.%3\.%4\.%5\.%6\.";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level7 lfo3
{mso-level-numbering:continue;
mso-level-text:"%1\.%2\.%3\.%4\.%5\.%6\.%7\.";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level8 lfo3
{mso-level-number-format:arabic;
mso-level-numbering:continue;
mso-level-text:"%1\.%2\.%3\.%4\.%5\.%6\.%7\.%8\.";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
@list l0:level9 lfo3
{mso-level-number-format:arabic;
mso-level-numbering:continue;
mso-level-text:"%1\.%2\.%3\.%4\.%5\.%6\.%7\.%8\.%9\.";
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:0cm;
text-indent:0cm;}
ol
{margin-bottom:0cm;}
ul
{margin-bottom:0cm;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang="EN-GB" link="#0563C1" vlink="#954F72">
<div class="WordSection1">
<p class="MsoNormal"><span lang="SV">Hi.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="SV"><o:p> </o:p></span></p>
<p class="MsoNormal">Have anyone here experienced having VERBS RDMA connection request rejects on Scale NSD servers with the error message “ibv_create_qp err 13”?<o:p></o:p></p>
<p class="MsoNormal">I’m having issues with this on a IBM ESS system.<o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal">The error mostly affects only one of the two GSSIO-nodes, and moves with the node even if I put all four of the infiniband links on the same infiniband switch as the working node is connected to.<o:p></o:p></p>
<p class="MsoNormal">The issue affects client nodes in different blade-chassis, going through different Infiniband swithes and cables, and also non-blade nodes running a slightly different os-setup and different infiniband HCAs.<o:p></o:p></p>
<p class="MsoNormal">MPI-jobs on the client nodes can communicate over the infiniband fabric without issues.<o:p></o:p></p>
<p class="MsoNormal">Upgrading all switches and HCAs to the latest firmware and making sure that client nodes have the same OFED-version as the ESS has had no impact on the issue.<o:p></o:p></p>
<p class="MsoNormal">When the issue is there, I can still do ibping between the nodes, ibroute gives me a working and correct path between the nodes that get connection rejects, and if I set up IPoIB, ip traffic works on the afflicted interfaces.<o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal">I have opened a PMR with IBM on the issue, so asking here is a parallel track for trying to find a solution to this.<o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal">Any help or suggestions is appreciated.<o:p></o:p></p>
<p class="MsoNormal">Regards,<o:p></o:p></p>
<p class="MsoNormal">Andreas Mattsson<o:p></o:p></p>
<p class="MsoNormal"><span style="font-size:12.0pt;font-family:"Times New Roman",serif;color:black;mso-fareast-language:EN-GB">_____________________________________________<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-family:"Lucida Grande";color:black;mso-fareast-language:EN-GB"><img width="121" height="65" style="width:1.2604in;height:.677in" id="_x0036_b0a11de-89d6-4863-b8e8-188f1e1c5a0c" src="cid:image001.png@01D36DC8.ACE08B40" alt="cid:part1.08040705.03090509@maxiv.lu.se"><br>
</span><b><span style="font-size:10.0pt;color:black;mso-fareast-language:EN-GB"> <br>
Andreas Mattsson<br>
</span></b><span style="font-size:8.0pt;color:black;mso-fareast-language:EN-GB">Systems Engineer</span><span style="font-size:12.0pt;font-family:"Times New Roman",serif;color:black;mso-fareast-language:EN-GB"><o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:12.0pt;font-family:"Times New Roman",serif;color:black;mso-fareast-language:EN-GB"><o:p> </o:p></span></p>
<p class="MsoNormal" style="margin-bottom:12.0pt"><span style="font-size:8.0pt;color:black;mso-fareast-language:EN-GB">MAX IV Laboratory<br>
Lund University<br>
</span><span style="font-size:8.5pt;font-family:"Times New Roman",serif;color:black;mso-fareast-language:EN-GB">P.O. Box 118, SE-221 00 Lund, Sweden<br>
Visiting address: Fotongatan 2, 225 94 Lund<br>
</span><span style="font-size:8.0pt;color:black;mso-fareast-language:EN-GB">Mobile: +46 706 64 95 44<br>
</span><span lang="SV" style="font-size:8.0pt;color:black;mso-fareast-language:EN-GB"><a href="mailto:andreas.mattsson@maxiv.se"><span lang="EN-GB" style="color:blue">andreas.mattsson@maxiv.se</span></a></span><span style="font-size:8.0pt;color:black;mso-fareast-language:EN-GB"><br>
</span><span lang="SV" style="font-size:8.0pt;color:black;mso-fareast-language:EN-GB"><a href="http://www.maxiv.se/"><span lang="EN-GB" style="color:blue">www.maxiv.se</span></a></span><span style="font-size:8.0pt;color:black;mso-fareast-language:EN-GB"><o:p></o:p></span></p>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
</body>
</html>