<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:x="urn:schemas-microsoft-com:office:excel" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<style><!--
/* Font Definitions */
@font-face
{font-family:SimSun;
panose-1:2 1 6 0 3 1 1 1 1 1;}
@font-face
{font-family:"MS Gothic";
panose-1:2 11 6 9 7 2 5 8 2 4;}
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:"Calibri Light";
panose-1:2 15 3 2 2 2 4 3 2 4;}
@font-face
{font-family:Consolas;
panose-1:2 11 6 9 2 2 4 3 2 4;}
@font-face
{font-family:"\@MS Gothic";
panose-1:2 11 6 9 7 2 5 8 2 4;}
@font-face
{font-family:"\@SimSun";
panose-1:2 1 6 0 3 1 1 1 1 1;}
@font-face
{font-family:Menlo;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0cm;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
h2
{mso-style-priority:9;
mso-style-link:"Heading 2 Char";
margin-top:2.0pt;
margin-right:0cm;
margin-bottom:0cm;
margin-left:0cm;
font-size:13.0pt;
font-family:"Calibri Light",sans-serif;
color:#2F5496;
font-weight:normal;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:#0563C1;
text-decoration:underline;}
pre
{mso-style-priority:99;
mso-style-link:"HTML Preformatted Char";
margin:0cm;
margin-bottom:.0001pt;
font-size:10.0pt;
font-family:"Courier New";}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
{mso-style-priority:34;
margin-top:0cm;
margin-right:0cm;
margin-bottom:0cm;
margin-left:36.0pt;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
p.Code, li.Code, div.Code
{mso-style-name:Code;
mso-style-link:"Code Char";
margin-top:0cm;
margin-right:0cm;
margin-bottom:0cm;
margin-left:36.0pt;
background:#FFF2CC;
font-size:11.0pt;
font-family:"Courier New";}
span.CodeChar
{mso-style-name:"Code Char";
mso-style-link:Code;
font-family:"Courier New";
background:#FFF2CC;}
span.Heading2Char
{mso-style-name:"Heading 2 Char";
mso-style-priority:9;
mso-style-link:"Heading 2";
font-family:"Calibri Light",sans-serif;
color:#2F5496;}
span.HTMLPreformattedChar
{mso-style-name:"HTML Preformatted Char";
mso-style-priority:99;
mso-style-link:"HTML Preformatted";
font-family:Consolas;}
p.code0, li.code0, div.code0
{mso-style-name:code;
margin-top:0cm;
margin-right:0cm;
margin-bottom:0cm;
margin-left:36.0pt;
background:#FFF2CC;
font-size:11.0pt;
font-family:"Courier New";}
span.codechar0
{mso-style-name:codechar;
font-family:"Courier New";
background:#FFF2CC;}
span.EmailStyle28
{mso-style-type:personal-reply;
font-family:"Calibri",sans-serif;
color:windowtext;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;}
@page WordSection1
{size:612.0pt 792.0pt;
margin:72.0pt 72.0pt 72.0pt 72.0pt;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:507064911;
mso-list-template-ids:1971634634;}
@list l1
{mso-list-id:948590699;
mso-list-template-ids:-1560760938;}
@list l1:level1
{mso-level-tab-stop:36.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l1:level2
{mso-level-tab-stop:72.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l1:level3
{mso-level-tab-stop:108.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l1:level4
{mso-level-tab-stop:144.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l1:level5
{mso-level-tab-stop:180.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l1:level6
{mso-level-tab-stop:216.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l1:level7
{mso-level-tab-stop:252.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l1:level8
{mso-level-tab-stop:288.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l1:level9
{mso-level-tab-stop:324.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l2
{mso-list-id:1675497088;
mso-list-template-ids:146418876;}
@list l2:level1
{mso-level-tab-stop:36.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l2:level2
{mso-level-tab-stop:72.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l2:level3
{mso-level-tab-stop:108.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l2:level4
{mso-level-tab-stop:144.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l2:level5
{mso-level-tab-stop:180.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l2:level6
{mso-level-tab-stop:216.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l2:level7
{mso-level-tab-stop:252.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l2:level8
{mso-level-tab-stop:288.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l2:level9
{mso-level-tab-stop:324.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l3
{mso-list-id:1871382398;
mso-list-template-ids:1446439018;}
@list l3:level1
{mso-level-tab-stop:36.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l3:level2
{mso-level-tab-stop:72.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l3:level3
{mso-level-tab-stop:108.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l3:level4
{mso-level-tab-stop:144.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l3:level5
{mso-level-tab-stop:180.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l3:level6
{mso-level-tab-stop:216.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l3:level7
{mso-level-tab-stop:252.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l3:level8
{mso-level-tab-stop:288.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l3:level9
{mso-level-tab-stop:324.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;}
ol
{margin-bottom:0cm;}
ul
{margin-bottom:0cm;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang="EN-GB" link="#0563C1" vlink="purple" style="word-wrap:break-word">
<div class="WordSection1">
<p class="MsoNormal"><span style="mso-fareast-language:EN-US">Just to close the loop on this, if anyone is having similar problems and wants to reference the cases:<o:p></o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US">IBM created efix 37 for 5.1.5.1 for this as
</span><span class="CodeChar"><span style="color:black">g515p1e37.x86_64-linux-gnr.tar.gz</span></span>
<span lang="EN-US">under </span><span style="mso-fareast-language:EN-US">TS012531429.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US">Lenovo have a copy of this and have provided update instructions for the DSSG under case #</span>
<span style="mso-fareast-language:EN-US">1001583845 (basically fail the recovery groups over and install the updated RPMs).<o:p></o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US">And this all appears to be working well for us now, so I can proceed with updating the other building blocks. A new version is due which will probably come out while I’m in the middle of doing that
of course…<o:p></o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US">Many thanks,<o:p></o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US">Luke<o:p></o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<div>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">-- <o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">Luke Sudbery<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">Principal Engineer (HPC and Storage).<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">Architecture, Infrastructure and Systems<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">Advanced Research Computing, IT Services<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">Room 132, Computer Centre G5, Elms Road<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><b><span style="font-size:9.0pt;color:#1F497D">Please note I don’t work on Monday.<o:p></o:p></span></b></p>
</div>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal"><b><span lang="EN-US">From:</span></b><span lang="EN-US"> gpfsug-discuss <gpfsug-discuss-bounces@gpfsug.org>
<b>On Behalf Of </b>Luke Sudbery<br>
<b>Sent:</b> 21 March 2023 13:47<br>
<b>To:</b> gpfsug main discussion list <gpfsug-discuss@gpfsug.org><br>
<b>Subject:</b> Re: [gpfsug-discuss] mmvdisk version/communication issues?<o:p></o:p></span></p>
</div>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US">Yes, that’s exactly what happened to me. If Lenovo had tested an upgrade they would have seen it too.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US">We have 3 DSSG, but they are all this cluster, and we don’t run GNR outside this cluster.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US">The 5.1.5.1 nodes can talk to each other OK. And to the downlevel nodes. But the downlevel nodes can’t talk to the newer nodes.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US">At present it doesn’t seem to present any major problem – but we haven’t had to move RGs around or do anything like that yet.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US">And we have a new DSSG to add to this cluster soon, and then some pool migrations. This needs to sorted before we can start that in confidence.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US">Many thanks,<o:p></o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US">Luke<o:p></o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<div>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">-- <o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">Luke Sudbery<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">Principal Engineer (HPC and Storage).<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">Architecture, Infrastructure and Systems<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">Advanced Research Computing, IT Services<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">Room 132, Computer Centre G5, Elms Road<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><b><span style="font-size:9.0pt;color:#1F497D">Please note I don’t work on Monday.<o:p></o:p></span></b></p>
</div>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal"><b><span lang="EN-US">From:</span></b><span lang="EN-US"> gpfsug-discuss <<a href="mailto:gpfsug-discuss-bounces@gpfsug.org">gpfsug-discuss-bounces@gpfsug.org</a>>
<b>On Behalf Of </b>Ryan Novosielski<br>
<b>Sent:</b> 21 March 2023 13:33<br>
<b>To:</b> gpfsug main discussion list <<a href="mailto:gpfsug-discuss@gpfsug.org">gpfsug-discuss@gpfsug.org</a>><br>
<b>Subject:</b> Re: [gpfsug-discuss] mmvdisk version/communication issues?<o:p></o:p></span></p>
</div>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
<table class="MsoNormalTable" border="0" cellspacing="0" cellpadding="0" align="left" width="100%" style="width:100.0%">
<tbody>
<tr>
<td width="10%" style="width:10.0%;border:solid #9C6500 1.0pt;background:#FFEB9C;padding:2.0pt 2.0pt 2.0pt 2.0pt">
<p class="MsoNormal" style="mso-element:frame;mso-element-frame-hspace:2.25pt;mso-element-wrap:around;mso-element-anchor-vertical:paragraph;mso-element-anchor-horizontal:column;mso-height-rule:exactly">
<b><span style="color:black">CAUTION:</span></b><span style="color:black"> This email originated from outside the organisation. Do not click links or open attachments unless you recognise the sender and know the content is safe.</span><o:p></o:p></p>
</td>
</tr>
</tbody>
</table>
<p class="MsoNormal"><o:p> </o:p></p>
<div>
<div>
<p class="MsoNormal">You know, I ran into this during the upgrade, didn’t have an explanation for it, and decided that I would see what happened when it was finished (problem was gone). I was going from DSS-G 3.2a to DSS-G 4.3a. It was at the stage where I
had one machine upgraded and one machine not upgraded, which makes sense. I believe I then tried it from the upgraded node, it worked fine, and I figured “well, I’m blowing that one away shortly anyway.”<o:p></o:p></p>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<p class="MsoNormal">Would one expect to only see this while nodes in the same cluster are not running the same version? I guess what I’m asking is would one expect to have that kind of traffic coming from any place else except the same cluster?<o:p></o:p></p>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<p class="MsoNormal">Sent from my iPhone<o:p></o:p></p>
</div>
<div>
<p class="MsoNormal" style="margin-bottom:12.0pt"><o:p> </o:p></p>
<blockquote style="margin-top:5.0pt;margin-bottom:5.0pt">
<p class="MsoNormal" style="margin-bottom:12.0pt">On Mar 21, 2023, at 08:38, Luke Sudbery <<a href="mailto:L.R.Sudbery@bham.ac.uk">L.R.Sudbery@bham.ac.uk</a>> wrote:<o:p></o:p></p>
</blockquote>
</div>
<blockquote style="margin-top:5.0pt;margin-bottom:5.0pt">
<div>
<p class="MsoNormal"> <o:p></o:p></p>
<p class="MsoNormal">Thank you, I’ll pass that onto Lenovo. I think I’ve also identified it as this in the fixies in 5.1.6.1 too:<o:p></o:p></p>
<p class="MsoNormal"><a href="https://www.ibm.com/support/pages/apar/IJ44607">https://www.ibm.com/support/pages/apar/IJ44607</a><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:24.0pt">IJ44607: SPECTRUM SCALE V5.1.3 AND GREATER FAILS TO INTEROPERATE WITH SPECTRUMSCALE VERSIONS PRIOR TO 5.1.3.</span></b><o:p></o:p></p>
<h2>Problem summary<o:p></o:p></h2>
<pre style="margin-left:36.0pt;text-indent:-18.0pt;mso-list:l2 level1 lfo2"><![if !supportLists]><span style="mso-list:Ignore">1.<span style="font:7.0pt "Times New Roman""> </span></span><![endif]>GNR RPCs fail when received by a GPFS daemon 5.1.3 or later<o:p></o:p></pre>
<pre style="margin-left:36.0pt;text-indent:-18.0pt;mso-list:l2 level1 lfo2"><![if !supportLists]><span style="mso-list:Ignore">2.<span style="font:7.0pt "Times New Roman""> </span></span><![endif]>from a GPFS daemon older than version 5.1.3.<o:p></o:p></pre>
<pre style="margin-left:36.0pt;text-indent:-18.0pt;mso-list:l2 level1 lfo2"><![if !supportLists]><span style="mso-list:Ignore">3.<span style="font:7.0pt "Times New Roman""> </span></span><![endif]>Kernel assert going off: privVfsP != NULL<o:p></o:p></pre>
<h2>Symptom:<o:p></o:p></h2>
<pre style="margin-left:36.0pt;text-indent:-18.0pt;mso-list:l1 level1 lfo4"><![if !supportLists]><span style="mso-list:Ignore">1.<span style="font:7.0pt "Times New Roman""> </span></span><![endif]>Hang in the command<o:p></o:p></pre>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">As such, AFAICT it will affect all Lenovo customers going from DSSG 3.x (GPFS 5.1.1.0) to 4.x (GPFS 5.1.5.1 efix20) and I’m a bit annoyed that didn’t pick it up before release.<o:p></o:p></p>
<p class="MsoNormal">Many thanks,<o:p></o:p></p>
<p class="MsoNormal"> <o:p></o:p></p>
<p class="MsoNormal">Luke<o:p></o:p></p>
<p class="MsoNormal"> <o:p></o:p></p>
<div>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">-- </span><o:p></o:p></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">Luke Sudbery</span><o:p></o:p></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">Principal Engineer (HPC and Storage).</span><o:p></o:p></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">Architecture, Infrastructure and Systems</span><o:p></o:p></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">Advanced Research Computing, IT Services</span><o:p></o:p></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D">Room 132, Computer Centre G5, Elms Road</span><o:p></o:p></p>
<p class="MsoNormal"><span style="font-size:9.0pt;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal"><b><span style="font-size:9.0pt;color:#1F497D">Please note I don’t work on Monday.</span></b><o:p></o:p></p>
</div>
<p class="MsoNormal"> <o:p></o:p></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal"><b><span lang="EN-US">From:</span></b><span lang="EN-US"> Gang Qiu <<a href="mailto:gangqiu@cn.ibm.com">gangqiu@cn.ibm.com</a>>
<br>
<b>Sent:</b> 21 March 2023 12:31<br>
<b>To:</b> Luke Sudbery (Advanced Research Computing) <<a href="mailto:l.r.sudbery@bham.ac.uk">l.r.sudbery@bham.ac.uk</a>>; gpfsug main discussion list <<a href="mailto:gpfsug-discuss@gpfsug.org">gpfsug-discuss@gpfsug.org</a>><br>
<b>Subject:</b> </span><span style="font-family:"MS Gothic"">答复</span><span lang="EN-US">: mmvdisk version/communication issues?</span><o:p></o:p></p>
</div>
</div>
<p class="MsoNormal"> <o:p></o:p></p>
<table class="MsoNormalTable" border="0" cellspacing="0" cellpadding="0" align="left" width="100%" style="width:100.0%">
<tbody>
<tr>
<td width="10%" style="width:10.0%;border:solid #9C6500 1.0pt;background:#FFEB9C;padding:2.0pt 2.0pt 2.0pt 2.0pt">
<p class="MsoNormal" style="mso-element:frame;mso-element-frame-hspace:2.25pt;mso-element-wrap:around;mso-element-anchor-vertical:paragraph;mso-element-anchor-horizontal:column;mso-height-rule:exactly">
<b><span style="color:black">CAUTION:</span></b><span style="color:black"> This email originated from outside the organisation. Do not click links or open attachments unless you recognise the sender and know the content is safe.</span><o:p></o:p></p>
</td>
</tr>
</tbody>
</table>
<p class="MsoNormal"><span style="mso-fareast-language:ZH-CN"> </span><o:p></o:p></p>
<div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;mso-fareast-language:ZH-CN"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;mso-fareast-language:ZH-CN"> </span><o:p></o:p></p>
<div>
<div>
<div>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">Luke,</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN"> </span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:9.0pt;font-family:Menlo;color:#0451A5;mso-fareast-language:ZH-CN">1.</span><span lang="EN-US" style="font-size:9.0pt;font-family:Menlo;color:black;mso-fareast-language:ZH-CN">
The case number is TS011014198 (Internal defect number is 1163600)</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:9.0pt;font-family:Menlo;color:black;mso-fareast-language:ZH-CN"> 5.1.3.1 efix47 includes this fix. (I didn't find the efix for 5.1.4 or 5.1.5)</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:9.0pt;font-family:Menlo;color:#0451A5;mso-fareast-language:ZH-CN">2.</span><span lang="EN-US" style="font-size:9.0pt;font-family:Menlo;color:black;mso-fareast-language:ZH-CN">
The impact is that any GNR-related command from 5.1.2 to 5.1.5 will hang.</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN"> </span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">Regards,</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">Gang Qiu</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">====================================================</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">Gang Qiu(</span><span lang="ZH-CN" style="font-size:10.5pt;font-family:SimSun;color:black;mso-fareast-language:ZH-CN">邱钢</span><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">)</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">Spectrum Scale Development - ECE</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">IBM China Systems Lab</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">Mobile: +86-18612867902</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">====================================================</span><o:p></o:p></p>
</div>
</div>
</div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;mso-fareast-language:ZH-CN"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;mso-fareast-language:ZH-CN"> </span><o:p></o:p></p>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal" style="margin-bottom:12.0pt"><b><span style="font-family:SimSun;color:black">发件人</span></b><b><span lang="EN-US" style="color:black">:</span></b><span lang="EN-US" style="color:black"> Luke Sudbery <<a href="mailto:l.r.sudbery@bham.ac.uk">l.r.sudbery@bham.ac.uk</a>><br>
</span><b><span style="font-family:SimSun;color:black">日期</span></b><b><span lang="EN-US" style="color:black">:</span></b><span lang="EN-US" style="color:black">
</span><span style="font-family:SimSun;color:black">星期二</span><span lang="EN-US" style="color:black">, 2023</span><span style="font-family:SimSun;color:black">年</span><span lang="EN-US" style="color:black">3</span><span style="font-family:SimSun;color:black">月</span><span lang="EN-US" style="color:black">21</span><span style="font-family:SimSun;color:black">日</span><span lang="EN-US" style="color:black">
17:31<br>
</span><b><span style="font-family:SimSun;color:black">收件人</span></b><b><span lang="EN-US" style="color:black">:</span></b><span lang="EN-US" style="color:black"> gpfsug main discussion list <<a href="mailto:gpfsug-discuss@gpfsug.org">gpfsug-discuss@gpfsug.org</a>>,
Gang Qiu <<a href="mailto:gangqiu@cn.ibm.com">gangqiu@cn.ibm.com</a>><br>
</span><b><span style="font-family:SimSun;color:black">主题</span></b><b><span lang="EN-US" style="color:black">:</span></b><span lang="EN-US" style="color:black"> [EXTERNAL] RE: mmvdisk version/communication issues?</span><o:p></o:p></p>
</div>
<div>
<p class="MsoNormal" style="mso-line-height-alt:.75pt"><span lang="EN-US" style="font-size:1.0pt;color:white">Thank you, that’s far more useful than Lenovo support have been so far! Unfortunately Lenovo only support particular versions of scale on their DSSG
and they are our only source of the required GNR packages. So a couple more questions if you
</span><o:p></o:p></p>
</div>
<div>
<p class="MsoNormal" style="mso-line-height-alt:.75pt"><span lang="EN-US" style="font-size:1.0pt;color:white">ZjQcmQRYFpfptBannerStart</span><o:p></o:p></p>
</div>
<table class="MsoNormalTable" border="0" cellspacing="0" cellpadding="0" width="100%" style="width:100.0%;border-radius:4px">
<tbody>
<tr>
<td style="padding:12.0pt 0cm 12.0pt 0cm">
<table class="MsoNormalTable" border="1" cellspacing="0" cellpadding="0" width="100%" style="width:100.0%;background:#D0D8DC;border:none;border-top:solid #90A4AE 3.0pt">
<tbody>
<tr>
<td valign="top" style="border:none;padding:0cm 7.5pt 3.75pt 4.5pt">
<table class="MsoNormalTable" border="0" cellspacing="0" cellpadding="0" align="left">
<tbody>
<tr>
<td style="padding:3.0pt 6.0pt 3.0pt 6.0pt">
<p class="MsoNormal"><b><span lang="EN-US" style="font-size:10.5pt;font-family:"Arial",sans-serif;color:black">This Message Is From an External Sender
</span></b><o:p></o:p></p>
</td>
</tr>
<tr>
<td style="padding:3.0pt 6.0pt 3.0pt 6.0pt">
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;font-family:"Arial",sans-serif;color:black">This message came from outside your organization.
</span><o:p></o:p></p>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
<div>
<p class="MsoNormal" style="mso-line-height-alt:.75pt"><span lang="EN-US" style="font-size:1.0pt;color:white">ZjQcmQRYFpfptBannerEnd</span><o:p></o:p></p>
</div>
<p class="MsoNormal"><span lang="EN-US">Thank you, that’s far more useful than Lenovo support have been so far!</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">Unfortunately Lenovo only support particular versions of scale on their DSSG and they are our only source of the required GNR packages.</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">So a couple more questions if you don’t mind!</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<ol style="margin-top:0cm" start="1" type="1">
<li class="MsoListParagraph" style="margin-left:0cm;mso-list:l3 level1 lfo7"><span lang="EN-US">Can you provide a link the APAR or similar I can share with Lenovo?</span><o:p></o:p></li><li class="MsoListParagraph" style="margin-left:0cm;mso-list:l3 level1 lfo7"><span lang="EN-US">Do you know of any workaround or other impact of this issue? (I think I’ve seen mmheath show false errors because of it). This may help us to decide whether to just
press ahead upgrading to 5.1.5 everywhere if the issue is not present 5.1.5 -> 5.1.5.</span><o:p></o:p></li></ol>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">Many thanks,</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">Luke</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">-- </span>
<o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Luke Sudbery</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Principal Engineer (HPC and Storage).</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Architecture, Infrastructure and Systems</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Advanced Research Computing, IT Services</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Room 132, Computer Centre G5, Elms Road</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal"><b><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Please note I don’t work on Monday.</span></b><o:p></o:p></p>
</div>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal"><b><span lang="EN-US">From:</span></b><span lang="EN-US"> gpfsug-discuss <<a href="mailto:gpfsug-discuss-bounces@gpfsug.org">gpfsug-discuss-bounces@gpfsug.org</a>>
<b>On Behalf Of </b>Gang Qiu<br>
<b>Sent:</b> 21 March 2023 03:49<br>
<b>To:</b> gpfsug main discussion list <<a href="mailto:gpfsug-discuss@gpfsug.org">gpfsug-discuss@gpfsug.org</a>><br>
<b>Subject:</b> [gpfsug-discuss] </span><span style="font-family:"MS Gothic"">答复</span><span lang="EN-US">: mmvdisk version/communication issues?</span><o:p></o:p></p>
</div>
</div>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<table class="MsoNormalTable" border="0" cellspacing="0" cellpadding="0" align="left" width="100%" style="width:100.0%">
<tbody>
<tr>
<td width="10%" style="width:10.0%;border:solid #9C6500 1.0pt;background:#FFEB9C;padding:2.0pt 2.0pt 2.0pt 2.0pt">
<p class="MsoNormal" style="mso-element:frame;mso-element-frame-hspace:2.25pt;mso-element-wrap:around;mso-element-anchor-vertical:paragraph;mso-element-anchor-horizontal:column;mso-height-rule:exactly">
<b><span lang="EN-US" style="color:black">CAUTION:</span></b><span lang="EN-US" style="color:black"> This email originated from outside the organisation. Do not click links or open attachments unless you recognise the sender and know the content is safe.</span><o:p></o:p></p>
</td>
</tr>
</tbody>
</table>
<p class="MsoNormal"><span lang="EN-US" style="mso-fareast-language:ZH-CN"> </span><o:p></o:p></p>
<div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;mso-fareast-language:ZH-CN"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;mso-fareast-language:ZH-CN"> </span><o:p></o:p></p>
<div>
<div>
<div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt">It is not mmvdisk issue. Mmvdisk hangs because ‘tslsrecgroup’ command hangs which executed by mmvdisk.</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt">I recreated it on my environment (coexist 5.1.3 or 5.1.4, 5.1.5 with 5.1.2 or previous version)</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt">After further investigation, it should be a known issue which is fixed in 5.1.6 PTF1.
</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt">To resolve this issue, we need to upgrade the version to 5.1.6.1 at least.
</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN"> </span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN"> </span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">Regards,</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">Gang Qiu</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">====================================================</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">Gang Qiu(</span><span lang="ZH-CN" style="font-size:10.5pt;font-family:SimSun;color:black;mso-fareast-language:ZH-CN">邱钢</span><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">)</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">Spectrum Scale Development - ECE</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">IBM China Systems Lab</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">Mobile: +86-18612867902</span><o:p></o:p></p>
<p class="MsoNormal" style="line-height:13.5pt;background:white"><span lang="EN-US" style="font-size:10.5pt;color:black;mso-fareast-language:ZH-CN">====================================================</span><o:p></o:p></p>
</div>
</div>
</div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;mso-fareast-language:ZH-CN"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;mso-fareast-language:ZH-CN"> </span><o:p></o:p></p>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal" style="margin-bottom:12.0pt"><b><span style="font-family:SimSun;color:black">发件人</span></b><b><span lang="EN-US" style="color:black">:</span></b><span lang="EN-US" style="color:black"> gpfsug-discuss <<a href="mailto:gpfsug-discuss-bounces@gpfsug.org">gpfsug-discuss-bounces@gpfsug.org</a>>
</span><span style="font-family:SimSun;color:black">代表</span><span lang="EN-US" style="color:black"> Luke Sudbery <<a href="mailto:l.r.sudbery@bham.ac.uk">l.r.sudbery@bham.ac.uk</a>><br>
</span><b><span style="font-family:SimSun;color:black">日期</span></b><b><span lang="EN-US" style="color:black">:</span></b><span lang="EN-US" style="color:black">
</span><span style="font-family:SimSun;color:black">星期六</span><span lang="EN-US" style="color:black">, 2023</span><span style="font-family:SimSun;color:black">年</span><span lang="EN-US" style="color:black">3</span><span style="font-family:SimSun;color:black">月</span><span lang="EN-US" style="color:black">18</span><span style="font-family:SimSun;color:black">日</span><span lang="EN-US" style="color:black">
00:06<br>
</span><b><span style="font-family:SimSun;color:black">收件人</span></b><b><span lang="EN-US" style="color:black">:</span></b><span lang="EN-US" style="color:black"> gpfsug main discussion list <<a href="mailto:gpfsug-discuss@gpfsug.org">gpfsug-discuss@gpfsug.org</a>><br>
</span><b><span style="font-family:SimSun;color:black">主题</span></b><b><span lang="EN-US" style="color:black">:</span></b><span lang="EN-US" style="color:black"> [EXTERNAL] Re: [gpfsug-discuss] mmvdisk version/communication issues?</span><o:p></o:p></p>
</div>
<div>
<p class="MsoNormal" style="mso-line-height-alt:.75pt"><span lang="EN-US" style="font-size:1.0pt;color:white">On further investigation the command does eventually complete, after 11 minutes rather than a couple of seconds. [root@ rds-pg-dssg01 ~]# time mmvdisk
pdisk list --rg rds_er_dssg02 --not-ok mmvdisk: All pdisks of recovery group 'rds_er_dssg02'
</span><o:p></o:p></p>
</div>
<div>
<p class="MsoNormal" style="mso-line-height-alt:.75pt"><span lang="EN-US" style="font-size:1.0pt;color:white">ZjQcmQRYFpfptBannerStart</span><o:p></o:p></p>
</div>
<table class="MsoNormalTable" border="0" cellspacing="0" cellpadding="0" width="100%" style="width:100.0%;border-radius:4px">
<tbody>
<tr>
<td style="padding:12.0pt 0cm 12.0pt 0cm">
<table class="MsoNormalTable" border="1" cellspacing="0" cellpadding="0" width="100%" style="width:100.0%;background:#D0D8DC;border:none;border-top:solid #90A4AE 3.0pt">
<tbody>
<tr>
<td valign="top" style="border:none;padding:0cm 7.5pt 3.75pt 4.5pt">
<table class="MsoNormalTable" border="0" cellspacing="0" cellpadding="0" align="left">
<tbody>
<tr>
<td style="padding:3.0pt 6.0pt 3.0pt 6.0pt">
<p class="MsoNormal"><b><span lang="EN-US" style="font-size:10.5pt;font-family:"Arial",sans-serif;color:black">This Message Is From an External Sender
</span></b><o:p></o:p></p>
</td>
</tr>
<tr>
<td style="padding:3.0pt 6.0pt 3.0pt 6.0pt">
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;font-family:"Arial",sans-serif;color:black">This message came from outside your organization.
</span><o:p></o:p></p>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
<div>
<p class="MsoNormal" style="mso-line-height-alt:.75pt"><span lang="EN-US" style="font-size:1.0pt;color:white">ZjQcmQRYFpfptBannerEnd</span><o:p></o:p></p>
</div>
<p class="MsoNormal"><span lang="EN-US">On further investigation the command does eventually complete, after 11 minutes rather than a couple of seconds.</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">[root@rds-pg-dssg01 ~]# time mmvdisk pdisk list --rg rds_er_dssg02 --not-ok</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">mmvdisk: All pdisks of recovery group 'rds_er_dssg02' are ok.</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black"> </span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">real 11m14.106s</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">user 0m1.430s</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">sys 0m0.555s</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">[root@rds-pg-dssg01 ~]#</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">Looking at the process tree, the bits that hang are:</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">[root@rds-pg-dssg01 ~]# time tslsrecgroup rds_er_dssg02 -Y --v2 --failure-domain</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">Failed to connect to file system daemon: Connection timed out</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black"> </span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">real 5m30.181s</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">user 0m0.001s</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">sys 0m0.003s</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">and then</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">[root@rds-pg-dssg01 ~]# time tslspdisk --recovery-group rds_er_dssg02 --notOK</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">Failed to connect to file system daemon: Connection timed out</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black"> </span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">real 5m30.247s</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">user 0m0.003s</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">sys 0m0.002s</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">[root@rds-pg-dssg01 ~]#</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">Which adds up to the 11 minutes.... then it does something else and just works. Or maybe it <i>doesn't work</i> and just wouldn't report any failed disks is there were any….</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">While hanging, the ts commands appear to be LISTENing, not attempting to make connections:</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">[root@rds-pg-dssg01 ~]# pidof tslspdisk</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">2156809</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">[root@rds-pg-dssg01 ~]# netstat -apt | grep 2156809</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">tcp 0 0 0.0.0.0:60000 0.0.0.0:* LISTEN 2156809/tslspdisk</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">[root@rds-pg-dssg01 ~]#</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">Port 60000 is the lowest of our </span><span class="codechar0"><span lang="EN-US" style="color:black">tscCmdPortRange</span></span><span lang="EN-US">.</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">Don’t know if that helps anyone….</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">Cheers,</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">Luke</span><o:p></o:p></p>
<div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">-- </span>
<o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Luke Sudbery</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Principal Engineer (HPC and Storage).</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Architecture, Infrastructure and Systems</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Advanced Research Computing, IT Services</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Room 132, Computer Centre G5, Elms Road</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal"><b><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Please note I don’t work on Monday.</span></b><o:p></o:p></p>
</div>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal"><b><span lang="EN-US">From:</span></b><span lang="EN-US"> gpfsug-discuss <<a href="mailto:gpfsug-discuss-bounces@gpfsug.org">gpfsug-discuss-bounces@gpfsug.org</a>>
<b>On Behalf Of </b>Luke Sudbery<br>
<b>Sent:</b> 17 March 2023 15:11<br>
<b>To:</b> <a href="mailto:gpfsug-discuss@gpfsug.org">gpfsug-discuss@gpfsug.org</a><br>
<b>Subject:</b> [gpfsug-discuss] mmvdisk version/communication issues?</span><o:p></o:p></p>
</div>
</div>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">Hello,</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">We 3 Lenovo DSSG “Building Blocks” as they call them – 2x GNR server pairs.</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">We’ve just upgraded the 1st of them from 3.2a (GPFS 5.1.1.0) to 4.3a (5.1.5.1 efix 20).</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">Now the older systems can’t communicated with the newer in certain circumstances, specifically querying recovery groups hosted on other servers.</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">It works old->old, new->old and new->new but not old->new.</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">If fairly sure it is not a TCP comms problem. I can ssh between the node as root and as the GPFS
</span><span class="codechar0"><span lang="EN-US" style="color:black">sudoUser</span></span><span lang="EN-US">. Port 1191 and the
</span><span class="codechar0"><span lang="EN-US" style="color:black">tscCmdPortRange</span></span><span lang="EN-US"> are open and accessible in both direction between the nodes. There are connections present between the nodes in netstat and in
</span><span class="codechar0"><span lang="EN-US" style="color:black">mmfsd.latest.log</span></span><span lang="EN-US">. No pending message (to that node) in
</span><span class="codechar0"><span lang="EN-US" style="color:black">mmdiag --network.</span></span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">In these examples </span><span class="codechar0"><span lang="EN-US" style="color:black">rds-er-dssg01/2</span></span><span lang="EN-US"> are upgraded,
</span><span class="codechar0"><span lang="EN-US" style="color:black">rds-pg-dssg01/2</span></span><span lang="EN-US"> are downlevel:</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">[root@rds-er-dssg01 ~]# mmvdisk pdisk list --rg rds_er_dssg02 --not-ok # New to new</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">mmvdisk: All pdisks of recovery group 'rds_er_dssg02' are ok.</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">[root@rds-er-dssg01 ~]# mmvdisk pdisk list --rg rds_pg_dssg02 --not-ok # New to old</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">mmvdisk: All pdisks of recovery group 'rds_pg_dssg02' are ok.</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">[root@rds-er-dssg01 ~]#</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">[root@rds-pg-dssg01 ~]# mmvdisk pdisk list --rg rds_pg_dssg02 --not-ok # Old to old</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">mmvdisk: All pdisks of recovery group 'rds_pg_dssg02' are ok.</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">[root@rds-pg-dssg01 ~]# mmvdisk pdisk list --rg rds_er_dssg02 --not-ok # Old to new [HANGS]</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">^Cmmvdisk: Command failed. Examine previous error messages to determine cause.</span><o:p></o:p></p>
<p class="code0"><span lang="EN-US" style="color:black">[root@rds-pg-dssg01 ~]#</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">Has anyone come across this? mmvdisk should work across slightly different versions of 5.1, right? No recovery group, cluster or filesystem versions have been changed yet.</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">We will also log a ticket with snaps and more info but wondered if anyone had seen this.</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">And while this particular command is not a major issue, we don’t know what else it may affect, before we proceed with the reset of the cluster.</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">Many thanks,</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US">Luke</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">-- </span>
<o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Luke Sudbery</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Principal Engineer (HPC and Storage).</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Architecture, Infrastructure and Systems</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Advanced Research Computing, IT Services</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Room 132, Computer Centre G5, Elms Road</span><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:9.0pt;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal"><b><span lang="EN-US" style="font-size:9.0pt;color:#1F497D">Please note I don’t work on Monday.</span></b><o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-US"> </span><o:p></o:p></p>
</div>
</div>
<p class="MsoNormal">_______________________________________________<br>
gpfsug-discuss mailing list<br>
gpfsug-discuss at gpfsug.org<br>
<a href="http://gpfsug.org/mailman/listinfo/gpfsug-discuss_gpfsug.org">http://gpfsug.org/mailman/listinfo/gpfsug-discuss_gpfsug.org</a><o:p></o:p></p>
</div>
</blockquote>
</div>
</div>
</body>
</html>