Running prologue on parent mom node: nid001325... Job 2269370.cbqs01 nodelist: nid[001325,001346-001349] CPU's online: 0-127 CPU's offline: 128-255 nid001347: CPU's online: 0-127 CPU's offline: 128-255 nid001348: CPU's online: 0-127 CPU's offline: 128-255 nid001346: CPU's online: 0-127 CPU's offline: 128-255 nid001349: CPU's online: 0-127 CPU's offline: 128-255 Job 2269370.cbqs01 - Prologue complete. Execution time: 8 seconds Sun Apr 24 16:17:13 UTC 2022 + export DATA=/lfs/f1/ops/prod/tmp/hmon2_06_prod + DATA=/lfs/f1/ops/prod/tmp/hmon2_06_prod + cyc=06 + export cycle=t06z + cycle=t06z + export jlogfile=/lfs/f1/ops/prod/tmp/hmon2_06_prod/jlogfile + jlogfile=/lfs/f1/ops/prod/tmp/hmon2_06_prod/jlogfile + mkdir -p /lfs/f1/ops/prod/tmp/hmon2_06_prod + cd /lfs/f1/ops/prod/tmp/hmon2_06_prod + export NET=hmon RUN=hmon + NET=hmon + RUN=hmon + export OMP_NUM_THREADS=1 + OMP_NUM_THREADS=1 + export OMP_STACKSIZE=128M + OMP_STACKSIZE=128M + export MKL_NUM_THREADS=1 + MKL_NUM_THREADS=1 + export HOMEhmon=/lfs/h1/ops/prod/packages/hmon.v3.2.0 + HOMEhmon=/lfs/h1/ops/prod/packages/hmon.v3.2.0 + export USHhmon=/lfs/h1/ops/prod/packages/hmon.v3.2.0/ush + USHhmon=/lfs/h1/ops/prod/packages/hmon.v3.2.0/ush + export EXhmon=/lfs/h1/ops/prod/packages/hmon.v3.2.0/scripts + EXhmon=/lfs/h1/ops/prod/packages/hmon.v3.2.0/scripts + export PARMhmon=/lfs/h1/ops/prod/packages/hmon.v3.2.0/parm + PARMhmon=/lfs/h1/ops/prod/packages/hmon.v3.2.0/parm + export FIXhmon=/lfs/h1/ops/prod/packages/hmon.v3.2.0/fix + FIXhmon=/lfs/h1/ops/prod/packages/hmon.v3.2.0/fix + export EXEChmon=/lfs/h1/ops/prod/packages/hmon.v3.2.0/exec + EXEChmon=/lfs/h1/ops/prod/packages/hmon.v3.2.0/exec + export WORKhmon=/lfs/f1/ops/prod/tmp/hmon2_06_prod + WORKhmon=/lfs/f1/ops/prod/tmp/hmon2_06_prod + source /lfs/h1/ops/prod/packages/hmon.v3.2.0/versions/run.ver ++ export PrgEnv_intel_ver=8.1.0 ++ PrgEnv_intel_ver=8.1.0 ++ export craype_ver=2.7.8 ++ craype_ver=2.7.8 ++ export intel_ver=19.1.3.304 ++ intel_ver=19.1.3.304 ++ export cray_mpich_ver=8.1.7 ++ cray_mpich_ver=8.1.7 ++ export cray_pals_ver=1.0.12 ++ cray_pals_ver=1.0.12 ++ export libjpeg_ver=9c ++ libjpeg_ver=9c ++ export grib_util_ver=1.2.3 ++ grib_util_ver=1.2.3 ++ export wgrib2_ver=2.0.8 ++ wgrib2_ver=2.0.8 ++ export python_ver=3.8.6 ++ python_ver=3.8.6 ++ export cfp_ver=2.0.4 ++ cfp_ver=2.0.4 ++ export atp_ver=3.14.3 ++ atp_ver=3.14.3 ++ export gempak_ver=7.14.1 ++ gempak_ver=7.14.1 ++ export netcdf_ver=4.7.4 ++ netcdf_ver=4.7.4 ++ export pnetcdf_ver=1.12.2 ++ pnetcdf_ver=1.12.2 ++ export hdf5_ver=1.10.6 ++ hdf5_ver=1.10.6 ++ export hwrf_ver=v13.2 ++ hwrf_ver=v13.2 ++ export gfs_ver=v16.2 ++ gfs_ver=v16.2 ++ export rtofs_ver=v2.2 ++ rtofs_ver=v2.2 ++ export PDY=20210824 ++ PDY=20210824 ++ export DCOMROOT=/lfs/h1/ops/canned/dcom ++ DCOMROOT=/lfs/h1/ops/canned/dcom ++ export COMINmsg=/lfs/h1/ops/canned/com/hwrf/v13.2/inphmon/test_2021082406 ++ COMINmsg=/lfs/h1/ops/canned/com/hwrf/v13.2/inphmon/test_2021082406 ++ [[ 20210824 == 20210904 ]] ++ export COMPATH=/lfs/h1/ops/canned/com/gfs:/lfs/h1/ops/canned/com/rtofs ++ COMPATH=/lfs/h1/ops/canned/com/gfs:/lfs/h1/ops/canned/com/rtofs +++ date '--date=20210824 06:00 UTC -6 hours' +%Y%m%d%H ++ priorymdh=2021082400 ++ export HISTDATA=/lfs/h1/ops/canned/com/hmon/v3.2/hmon.2021082400 ++ HISTDATA=/lfs/h1/ops/canned/com/hmon/v3.2/hmon.2021082400 ++ export DBNLOG=YES ++ DBNLOG=YES ++ export PARATEST=YES ++ PARATEST=YES /lfs/h1/ops/prod/packages/hmon.v3.2.0/versions/run.ver: line 46: ECF_NAME: unbound variable CPU's online: 0-127 CPU's offline: 128-255 nid001347: CPU's online: 0-127 CPU's offline: 128-255 nid001348: CPU's online: 0-127 CPU's offline: 128-255 nid001346: CPU's online: 0-127 CPU's offline: 128-255 nid001349: CPU's online: 0-127 CPU's offline: 128-255 Job 2269370.cbqs01 - Epilogue complete. Execution time: 10 seconds ================================================================== BEGIN - DEBUG INFO ================================================================== Job 2269370.cbqs01 - Post Job Mem Usage: nid001325: 2022-04-24.07 1 =========================================== nid001325: 2022-04-24.07 2 #### Node Memory Usage for nid001325 #### nid001325: 2022-04-24.07 3 ------------------ nid001325: 2022-04-24.07 4 Mem: total:527077124 used:7930728 free:515855736 shared:1962720 cache:3290660 avail:515137468 nid001325: 2022-04-24.07 5 0 /dev/shm nid001325: 2022-04-24.07 6 45M /tmp nid001325: 2022-04-24.07 7 =========================================== nid001325: 2022-04-24.14 1 =========================================== nid001325: 2022-04-24.14 2 #### Node Memory Usage for nid001325 #### nid001325: 2022-04-24.14 3 ------------------ nid001325: 2022-04-24.14 4 Mem: total:527077124 used:7936912 free:515825248 shared:1962732 cache:3314964 avail:515119204 nid001325: 2022-04-24.14 5 0 /dev/shm nid001325: 2022-04-24.14 6 45M /tmp nid001325: 2022-04-24.14 7 =========================================== nid001346: 2022-04-24.08 1 =========================================== nid001346: 2022-04-24.08 2 #### Node Memory Usage for nid001346 #### nid001346: 2022-04-24.08 3 ------------------ nid001346: 2022-04-24.08 4 Mem: total:527077124 used:7984988 free:515854776 shared:1928460 cache:3237360 avail:515126336 nid001346: 2022-04-24.08 5 0 /dev/shm nid001346: 2022-04-24.08 6 40M /tmp nid001346: 2022-04-24.08 7 =========================================== nid001346: 2022-04-24.14 1 =========================================== nid001346: 2022-04-24.14 2 #### Node Memory Usage for nid001346 #### nid001346: 2022-04-24.14 3 ------------------ nid001346: 2022-04-24.14 4 Mem: total:527077124 used:7996856 free:515822060 shared:1928516 cache:3258208 avail:515103536 nid001346: 2022-04-24.14 5 0 /dev/shm nid001346: 2022-04-24.14 6 40M /tmp nid001346: 2022-04-24.14 7 =========================================== nid001347: 2022-04-24.08 1 =========================================== nid001347: 2022-04-24.08 2 #### Node Memory Usage for nid001347 #### nid001347: 2022-04-24.08 3 ------------------ nid001347: 2022-04-24.08 4 Mem: total:527077124 used:8144140 free:515655936 shared:1938456 cache:3277048 avail:514942600 nid001347: 2022-04-24.08 5 0 /dev/shm nid001347: 2022-04-24.08 6 42M /tmp nid001347: 2022-04-24.08 7 =========================================== nid001347: 2022-04-24.14 1 =========================================== nid001347: 2022-04-24.14 2 #### Node Memory Usage for nid001347 #### nid001347: 2022-04-24.14 3 ------------------ nid001347: 2022-04-24.14 4 Mem: total:527077124 used:8157216 free:515618980 shared:1938564 cache:3300928 avail:514917884 nid001347: 2022-04-24.14 5 0 /dev/shm nid001347: 2022-04-24.14 6 42M /tmp nid001347: 2022-04-24.14 7 =========================================== nid001348: 2022-04-24.08 1 =========================================== nid001348: 2022-04-24.08 2 #### Node Memory Usage for nid001348 #### nid001348: 2022-04-24.08 3 ------------------ nid001348: 2022-04-24.08 4 Mem: total:527077124 used:7932396 free:515927472 shared:1928580 cache:3217256 avail:515189300 nid001348: 2022-04-24.08 5 0 /dev/shm nid001348: 2022-04-24.08 6 40M /tmp nid001348: 2022-04-24.08 7 =========================================== nid001348: 2022-04-24.14 1 =========================================== nid001348: 2022-04-24.14 2 #### Node Memory Usage for nid001348 #### nid001348: 2022-04-24.14 3 ------------------ nid001348: 2022-04-24.14 4 Mem: total:527077124 used:7942100 free:515893160 shared:1928688 cache:3241864 avail:515167360 nid001348: 2022-04-24.14 5 0 /dev/shm nid001348: 2022-04-24.14 6 40M /tmp nid001348: 2022-04-24.14 7 =========================================== nid001349: 2022-04-24.08 1 =========================================== nid001349: 2022-04-24.08 2 #### Node Memory Usage for nid001349 #### nid001349: 2022-04-24.08 3 ------------------ nid001349: 2022-04-24.08 4 Mem: total:527077124 used:8073236 free:515725764 shared:1970100 cache:3278124 avail:514997648 nid001349: 2022-04-24.08 5 0 /dev/shm nid001349: 2022-04-24.08 6 42M /tmp nid001349: 2022-04-24.08 7 =========================================== nid001349: 2022-04-24.14 1 =========================================== nid001349: 2022-04-24.14 2 #### Node Memory Usage for nid001349 #### nid001349: 2022-04-24.14 3 ------------------ nid001349: 2022-04-24.14 4 Mem: total:527077124 used:8090204 free:515684244 shared:1970180 cache:3302676 avail:514968360 nid001349: 2022-04-24.14 5 0 /dev/shm nid001349: 2022-04-24.14 6 42M /tmp nid001349: 2022-04-24.14 7 =========================================== ------------------------------------------------------------------ Job 2269370.cbqs01 - /var/log/messages for job duration: nid001325: 2022-04-24A--:--:--.------+--:-- nid001325 #### nid001325 - Job 2269370.cbqs01 Runtime Data from /var/log/messages nid001325: 2022-04-24T16:17:05.844469+00:00 nid001325 prologue: MARK: Job 2269370.cbqs01 Start nid001325: 2022-04-24T16:17:06.178916+00:00 nid001325 prologue: Job 2269370.cbqs01 nodelist: nid[001325,001346-001349] nid001325: 2022-04-24T16:17:06.180555+00:00 nid001325 prologue: Job 2269370.cbqs01 - Checking existing ASLR settings... nid001325: 2022-04-24T16:17:06.519637+00:00 nid001325 prologue: Job 2269370.cbqs01 - Checking palsd open file count... nid001325: 2022-04-24T16:17:07.217342+00:00 nid001325 prologue: Job 2269370.cbqs01 - Recording pre-job HSN counters... nid001325: 2022-04-24T16:17:07.730492+00:00 nid001325 prologue: Job 2269370.cbqs01 - Recording pre-job memory usage... nid001325: 2022-04-24T16:17:08.114746+00:00 nid001325 prologue: Job 2269370.cbqs01 - Killing any stray user processes... nid001325: 2022-04-24T16:17:08.713268+00:00 nid001325 prologue: Job 2269370.cbqs01 - Enabling turboboost... nid001325: 2022-04-24T16:17:09.050305+00:00 nid001325 prologue: Job 2269370.cbqs01 - Disabling hyperthreading... nid001325: 2022-04-24T16:17:09.054611+00:00 nid001325 toggleHT: HyperThreading is already disabled on this node: 0-127\n nid001325: 2022-04-24T16:17:10.058180+00:00 nid001325 prologue: Job 2269370.cbqs01 - HUPing pbs_mom... nid001325: 2022-04-24T16:17:11.813192+00:00 nid001325 prologue: Job 2269370.cbqs01 - Verifying post-boot workarounds... nid001325: 2022-04-24T16:17:12.730772+00:00 nid001325 prologue: Job 2269370.cbqs01 - Warchk Complete nid001325: 2022-04-24T16:17:12.731825+00:00 nid001325 prologue: Job 2269370.cbqs01 - Recording initial NFS client statistics... nid001325: 2022-04-24T16:17:13.402920+00:00 nid001325 prologue: Job 2269370.cbqs01 - Prologue complete. Execution time: 8 seconds nid001325: 2022-04-24T16:17:13.751038+00:00 nid001325 epilogue: Job 2269370.cbqs01 complete, running post-job actions. nid001325: 2022-04-24T16:17:13.764259+00:00 nid001325 epilogue: Job 2269370.cbqs01 - Recording post-job HSN counters... nid001325: 2022-04-24T16:17:14.255331+00:00 nid001325 epilogue: Job 2269370.cbqs01 - Recording post-job memory usage... nid001325: 2022-04-24T16:17:14.618881+00:00 nid001325 epilogue: Job 2269370.cbqs01 - Recording job NFS statistics to /tmp/nfsstats.2269370.cbqs01 nid001325: 2022-04-24T16:17:16.112216+00:00 nid001325 epilogue: Job 2269370.cbqs01 - Clearing /tmp... nid001325: 2022-04-24T16:17:16.509562+00:00 nid001325 epilogue: Job 2269370.cbqs01 - Clearing shared memory... nid001325: 2022-04-24T16:17:16.864546+00:00 nid001325 epilogue: Job 2269370.cbqs01 - Clearing memory cache... nid001325: 2022-04-24T16:17:17.076419+00:00 nid001325 kernel: [3240257.975128] drop_caches (137480): drop_caches: 3 nid001325: 2022-04-24T16:17:18.347375+00:00 nid001325 epilogue: Job 2269370.cbqs01 - Releasing Lustre Locks... nid001325: 2022-04-24T16:17:19.737872+00:00 nid001325 epilogue: MARK: Job 2269370.cbqs01 Complete. nid001346: 2022-04-24A--:--:--.------+--:-- nid001346 #### nid001346 - Job 2269370.cbqs01 Runtime Data from /var/log/messages nid001346: 2022-04-24T16:17:06.171057+00:00 nid001346 prologue: MARK: Job 2269370.cbqs01 Start nid001346: 2022-04-24T16:17:06.173075+00:00 nid001346 systemd[1]: session-127109.scope: Succeeded. nid001346: 2022-04-24T16:17:06.517754+00:00 nid001346 systemd[1]: session-127110.scope: Succeeded. nid001346: 2022-04-24T16:17:07.204288+00:00 nid001346 systemd[1]: session-127111.scope: Succeeded. nid001346: 2022-04-24T16:17:07.703913+00:00 nid001346 systemd[1]: session-127112.scope: Succeeded. nid001346: 2022-04-24T16:17:08.113293+00:00 nid001346 systemd[1]: session-127113.scope: Succeeded. nid001346: 2022-04-24T16:17:08.703977+00:00 nid001346 systemd[1]: session-127114.scope: Succeeded. nid001346: 2022-04-24T16:17:09.037029+00:00 nid001346 systemd[1]: session-127115.scope: Succeeded. nid001346: 2022-04-24T16:17:10.430280+00:00 nid001346 toggleHT: HyperThreading is already disabled on this node: 0-127\n nid001346: 2022-04-24T16:17:10.434359+00:00 nid001346 systemd[1]: session-127116.scope: Succeeded. nid001346: 2022-04-24T16:17:11.802950+00:00 nid001346 systemd[1]: session-127117.scope: Succeeded. nid001346: 2022-04-24T16:17:12.718427+00:00 nid001346 systemd[1]: session-127118.scope: Succeeded. nid001346: 2022-04-24T16:17:13.061552+00:00 nid001346 systemd[1]: session-127119.scope: Succeeded. nid001346: 2022-04-24T16:17:13.399749+00:00 nid001346 systemd[1]: session-127120.scope: Succeeded. nid001346: 2022-04-24T16:17:14.245715+00:00 nid001346 systemd[1]: session-127121.scope: Succeeded. nid001346: 2022-04-24T16:17:14.615061+00:00 nid001346 systemd[1]: session-127122.scope: Succeeded. nid001346: 2022-04-24T16:17:16.110482+00:00 nid001346 systemd[1]: session-127123.scope: Succeeded. nid001346: 2022-04-24T16:17:16.497491+00:00 nid001346 systemd[1]: session-127124.scope: Succeeded. nid001346: 2022-04-24T16:17:16.862284+00:00 nid001346 systemd[1]: session-127125.scope: Succeeded. nid001346: 2022-04-24T16:17:17.801336+00:00 nid001346 kernel: [3240389.979858] drop_caches (88408): drop_caches: 3 nid001346: 2022-04-24T16:17:18.351797+00:00 nid001346 systemd[1]: session-127126.scope: Succeeded. nid001346: 2022-04-24T16:17:19.681488+00:00 nid001346 systemd[1]: session-127127.scope: Succeeded. nid001346: 2022-04-24T16:17:20.068081+00:00 nid001346 epilogue: MARK: Job 2269370.cbqs01 Complete. nid001347: 2022-04-24A--:--:--.------+--:-- nid001347 #### nid001347 - Job 2269370.cbqs01 Runtime Data from /var/log/messages nid001347: 2022-04-24T16:17:06.171512+00:00 nid001347 prologue: MARK: Job 2269370.cbqs01 Start nid001347: 2022-04-24T16:17:06.174097+00:00 nid001347 systemd[1]: session-126919.scope: Succeeded. nid001347: 2022-04-24T16:17:06.512682+00:00 nid001347 systemd[1]: session-126920.scope: Succeeded. nid001347: 2022-04-24T16:17:07.211132+00:00 nid001347 systemd[1]: session-126921.scope: Succeeded. nid001347: 2022-04-24T16:17:07.708732+00:00 nid001347 systemd[1]: session-126922.scope: Succeeded. nid001347: 2022-04-24T16:17:08.101625+00:00 nid001347 systemd[1]: session-126923.scope: Succeeded. nid001347: 2022-04-24T16:17:08.694983+00:00 nid001347 systemd[1]: session-126924.scope: Succeeded. nid001347: 2022-04-24T16:17:09.040688+00:00 nid001347 systemd[1]: session-126925.scope: Succeeded. nid001347: 2022-04-24T16:17:10.425202+00:00 nid001347 toggleHT: HyperThreading is already disabled on this node: 0-127\n nid001347: 2022-04-24T16:17:10.429629+00:00 nid001347 systemd[1]: session-126926.scope: Succeeded. nid001347: 2022-04-24T16:17:11.795554+00:00 nid001347 systemd[1]: session-126927.scope: Succeeded. nid001347: 2022-04-24T16:17:12.721005+00:00 nid001347 systemd[1]: session-126928.scope: Succeeded. nid001347: 2022-04-24T16:17:13.067829+00:00 nid001347 systemd[1]: session-126929.scope: Succeeded. nid001347: 2022-04-24T16:17:13.391120+00:00 nid001347 systemd[1]: session-126930.scope: Succeeded. nid001347: 2022-04-24T16:17:14.247428+00:00 nid001347 systemd[1]: session-126931.scope: Succeeded. nid001347: 2022-04-24T16:17:14.610669+00:00 nid001347 systemd[1]: session-126932.scope: Succeeded. nid001347: 2022-04-24T16:17:16.038221+00:00 nid001347 systemd[1]: session-126933.scope: Succeeded. nid001347: 2022-04-24T16:17:16.502866+00:00 nid001347 systemd[1]: session-126934.scope: Succeeded. nid001347: 2022-04-24T16:17:16.855301+00:00 nid001347 systemd[1]: session-126935.scope: Succeeded. nid001347: 2022-04-24T16:17:17.773311+00:00 nid001347 kernel: [3240406.100884] drop_caches (246461): drop_caches: 3 nid001347: 2022-04-24T16:17:18.294032+00:00 nid001347 systemd[1]: session-126936.scope: Succeeded. nid001347: 2022-04-24T16:17:19.735991+00:00 nid001347 systemd[1]: session-126937.scope: Succeeded. nid001347: 2022-04-24T16:17:20.061982+00:00 nid001347 epilogue: MARK: Job 2269370.cbqs01 Complete. nid001348: 2022-04-24A--:--:--.------+--:-- nid001348 #### nid001348 - Job 2269370.cbqs01 Runtime Data from /var/log/messages nid001348: 2022-04-24T16:17:06.175278+00:00 nid001348 prologue: MARK: Job 2269370.cbqs01 Start nid001348: 2022-04-24T16:17:06.177068+00:00 nid001348 systemd[1]: session-126999.scope: Succeeded. nid001348: 2022-04-24T16:17:06.514493+00:00 nid001348 systemd[1]: session-127000.scope: Succeeded. nid001348: 2022-04-24T16:17:07.153826+00:00 nid001348 systemd[1]: session-127001.scope: Succeeded. nid001348: 2022-04-24T16:17:07.729195+00:00 nid001348 systemd[1]: session-127002.scope: Succeeded. nid001348: 2022-04-24T16:17:08.108136+00:00 nid001348 systemd[1]: session-127003.scope: Succeeded. nid001348: 2022-04-24T16:17:08.600705+00:00 nid001348 systemd[1]: session-127004.scope: Succeeded. nid001348: 2022-04-24T16:17:09.048819+00:00 nid001348 systemd[1]: session-127005.scope: Succeeded. nid001348: 2022-04-24T16:17:10.428726+00:00 nid001348 toggleHT: HyperThreading is already disabled on this node: 0-127\n nid001348: 2022-04-24T16:17:10.432309+00:00 nid001348 systemd[1]: session-127006.scope: Succeeded. nid001348: 2022-04-24T16:17:11.804778+00:00 nid001348 systemd[1]: session-127007.scope: Succeeded. nid001348: 2022-04-24T16:17:12.654689+00:00 nid001348 systemd[1]: session-127008.scope: Succeeded. nid001348: 2022-04-24T16:17:13.070506+00:00 nid001348 systemd[1]: session-127009.scope: Succeeded. nid001348: 2022-04-24T16:17:13.399755+00:00 nid001348 systemd[1]: session-127010.scope: Succeeded. nid001348: 2022-04-24T16:17:14.247023+00:00 nid001348 systemd[1]: session-127011.scope: Succeeded. nid001348: 2022-04-24T16:17:14.611968+00:00 nid001348 systemd[1]: session-127012.scope: Succeeded. nid001348: 2022-04-24T16:17:15.908172+00:00 nid001348 systemd[1]: session-127013.scope: Succeeded. nid001348: 2022-04-24T16:17:16.505301+00:00 nid001348 systemd[1]: session-127014.scope: Succeeded. nid001348: 2022-04-24T16:17:16.860767+00:00 nid001348 systemd[1]: session-127015.scope: Succeeded. nid001348: 2022-04-24T16:17:17.783538+00:00 nid001348 kernel: [3240400.279182] drop_caches (113808): drop_caches: 3 nid001348: 2022-04-24T16:17:17.959495+00:00 nid001348 systemd[1]: session-127016.scope: Succeeded. nid001348: 2022-04-24T16:17:19.154486+00:00 nid001348 systemd[1]: session-127017.scope: Succeeded. nid001348: 2022-04-24T16:17:20.070393+00:00 nid001348 epilogue: MARK: Job 2269370.cbqs01 Complete. nid001349: 2022-04-24A--:--:--.------+--:-- nid001349 #### nid001349 - Job 2269370.cbqs01 Runtime Data from /var/log/messages nid001349: 2022-04-24T16:17:06.173881+00:00 nid001349 prologue: MARK: Job 2269370.cbqs01 Start nid001349: 2022-04-24T16:17:06.175964+00:00 nid001349 systemd[1]: session-127578.scope: Succeeded. nid001349: 2022-04-24T16:17:06.516104+00:00 nid001349 systemd[1]: session-127579.scope: Succeeded. nid001349: 2022-04-24T16:17:07.216278+00:00 nid001349 systemd[1]: session-127580.scope: Succeeded. nid001349: 2022-04-24T16:17:07.720000+00:00 nid001349 systemd[1]: session-127581.scope: Succeeded. nid001349: 2022-04-24T16:17:08.109271+00:00 nid001349 systemd[1]: session-127582.scope: Succeeded. nid001349: 2022-04-24T16:17:08.711538+00:00 nid001349 systemd[1]: session-127583.scope: Succeeded. nid001349: 2022-04-24T16:17:09.042825+00:00 nid001349 systemd[1]: session-127584.scope: Succeeded. nid001349: 2022-04-24T16:17:10.432818+00:00 nid001349 toggleHT: HyperThreading is already disabled on this node: 0-127\n nid001349: 2022-04-24T16:17:10.436584+00:00 nid001349 systemd[1]: session-127585.scope: Succeeded. nid001349: 2022-04-24T16:17:11.811377+00:00 nid001349 systemd[1]: session-127586.scope: Succeeded. nid001349: 2022-04-24T16:17:12.727066+00:00 nid001349 systemd[1]: session-127587.scope: Succeeded. nid001349: 2022-04-24T16:17:13.068106+00:00 nid001349 systemd[1]: session-127588.scope: Succeeded. nid001349: 2022-04-24T16:17:13.400121+00:00 nid001349 systemd[1]: session-127589.scope: Succeeded. nid001349: 2022-04-24T16:17:14.253729+00:00 nid001349 systemd[1]: session-127590.scope: Succeeded. nid001349: 2022-04-24T16:17:14.617338+00:00 nid001349 systemd[1]: session-127591.scope: Succeeded. nid001349: 2022-04-24T16:17:16.091807+00:00 nid001349 systemd[1]: session-127592.scope: Succeeded. nid001349: 2022-04-24T16:17:16.508442+00:00 nid001349 systemd[1]: session-127593.scope: Succeeded. nid001349: 2022-04-24T16:17:16.863015+00:00 nid001349 systemd[1]: session-127594.scope: Succeeded. nid001349: 2022-04-24T16:17:17.797338+00:00 nid001349 kernel: [3240435.132107] drop_caches (80070): drop_caches: 3 nid001349: 2022-04-24T16:17:18.381900+00:00 nid001349 systemd[1]: session-127595.scope: Succeeded. nid001349: 2022-04-24T16:17:19.720092+00:00 nid001349 systemd[1]: session-127596.scope: Succeeded. nid001349: 2022-04-24T16:17:20.069586+00:00 nid001349 epilogue: MARK: Job 2269370.cbqs01 Complete. ------------------------------------------------------------------ Job 2269370.cbqs01 - dmesg output for job duration: nid001325: 2022-04-24A--:--:--.------+--:-- nid001325 #### nid001325 - Job 2269370.cbqs01 Runtime Data from dmesg nid001325: [Sun Apr 24 16:13:47 2022] MARK: Job 2269370.cbqs01 Start nid001325: [Sun Apr 24 16:13:58 2022] drop_caches (137480): drop_caches: 3 nid001325: [Sun Apr 24 16:14:01 2022] MARK: Job 2269370.cbqs01 Complete. nid001346: 2022-04-24A--:--:--.------+--:-- nid001346 #### nid001346 - Job 2269370.cbqs01 Runtime Data from dmesg nid001346: [Sun Apr 24 16:15:56 2022] MARK: Job 2269370.cbqs01 Start nid001346: [Sun Apr 24 16:16:07 2022] drop_caches (88408): drop_caches: 3 nid001346: [Sun Apr 24 16:16:10 2022] MARK: Job 2269370.cbqs01 Complete nid001347: 2022-04-24A--:--:--.------+--:-- nid001347 #### nid001347 - Job 2269370.cbqs01 Runtime Data from dmesg nid001347: [Sun Apr 24 16:16:03 2022] MARK: Job 2269370.cbqs01 Start nid001347: [Sun Apr 24 16:16:15 2022] drop_caches (246461): drop_caches: 3 nid001347: [Sun Apr 24 16:16:17 2022] MARK: Job 2269370.cbqs01 Complete nid001348: 2022-04-24A--:--:--.------+--:-- nid001348 #### nid001348 - Job 2269370.cbqs01 Runtime Data from dmesg nid001348: [Sun Apr 24 16:15:55 2022] MARK: Job 2269370.cbqs01 Start nid001348: [Sun Apr 24 16:16:07 2022] drop_caches (113808): drop_caches: 3 nid001348: [Sun Apr 24 16:16:09 2022] MARK: Job 2269370.cbqs01 Complete nid001349: 2022-04-24A--:--:--.------+--:-- nid001349 #### nid001349 - Job 2269370.cbqs01 Runtime Data from dmesg nid001349: [Sun Apr 24 16:16:21 2022] MARK: Job 2269370.cbqs01 Start nid001349: [Sun Apr 24 16:16:33 2022] drop_caches (80070): drop_caches: 3 nid001349: [Sun Apr 24 16:16:35 2022] MARK: Job 2269370.cbqs01 Complete ------------------------------------------------------------------ Job 2269370.cbqs01 - Pre/Post job diff on HSN (MLX) Counters: nid001325: 2022-04-24A--:--:--.------+--:-- nid001325 #### nid001325 - Job 2269370.cbqs01 HSN0 MLX Counter Post-Job Difference nid001325: port_rcv_data: 281259147256711 | port_rcv_data: 281259147262739 nid001325: port_rcv_packets: 526774115600 | port_rcv_packets: 526774115658 nid001325: port_xmit_data: 291513703030340 | port_xmit_data: 291513703036160 nid001325: port_xmit_packets: 533733394694 | port_xmit_packets: 533733394752 nid001325: rx_bytes: 132292300867 | rx_bytes: 132292632527 nid001325: rx_packets: 89101868 | rx_packets: 89103078 nid001325: tx_bytes: 58334811987 | tx_bytes: 58335055568 nid001325: tx_packets: 58949063 | tx_packets: 58950350 nid001325: unicast_rcv_packets: 526774115600 | unicast_rcv_packets: 526774115658 nid001325: unicast_xmit_packets: 533733394694 | unicast_xmit_packets: 533733394752 nid001346: 2022-04-24A--:--:--.------+--:-- nid001346 #### nid001346 - Job 2269370.cbqs01 HSN0 MLX Counter Post-Job Difference nid001346: multicast: 5951529 | multicast: 5951530 nid001346: port_rcv_data: 276943946016166 | port_rcv_data: 276943946016429 nid001346: port_rcv_packets: 523172083391 | port_rcv_packets: 523172083397 nid001346: port_xmit_data: 282137022007599 | port_xmit_data: 282137022007884 nid001346: port_xmit_packets: 529608929200 | port_xmit_packets: 529608929206 nid001346: rx_bytes: 131129162066 | rx_bytes: 131129409162 nid001346: rx_packets: 71151386 | rx_packets: 71152225 nid001346: tx_bytes: 41908960703 | tx_bytes: 41909154094 nid001346: tx_packets: 42962588 | tx_packets: 42963896 nid001346: unicast_rcv_packets: 523172083391 | unicast_rcv_packets: 523172083397 nid001346: unicast_xmit_packets: 529608929200 | unicast_xmit_packets: 529608929206 nid001347: 2022-04-24A--:--:--.------+--:-- nid001347 #### nid001347 - Job 2269370.cbqs01 HSN0 MLX Counter Post-Job Difference nid001347: multicast: 5951985 | multicast: 5951987 nid001347: port_rcv_data: 282241877944670 | port_rcv_data: 282241877946214 nid001347: port_rcv_packets: 480607263383 | port_rcv_packets: 480607263453 nid001347: port_xmit_data: 279877998336833 | port_xmit_data: 279877998338377 nid001347: port_xmit_packets: 479234407433 | port_xmit_packets: 479234407503 nid001347: rx_bytes: 129725348397 | rx_bytes: 129725595438 nid001347: rx_packets: 77180374 | rx_packets: 77181212 nid001347: tx_bytes: 39289037978 | tx_bytes: 39289232505 nid001347: tx_packets: 49189239 | tx_packets: 49190505 nid001347: unicast_rcv_packets: 480607263383 | unicast_rcv_packets: 480607263453 nid001347: unicast_xmit_packets: 479234407433 | unicast_xmit_packets: 479234407503 nid001348: 2022-04-24A--:--:--.------+--:-- nid001348 #### nid001348 - Job 2269370.cbqs01 HSN0 MLX Counter Post-Job Difference nid001348: multicast: 5952784 | multicast: 5952785 nid001348: rx_bytes: 136463758923 | rx_bytes: 136464005623 nid001348: rx_packets: 110946998 | rx_packets: 110947831 nid001348: tx_bytes: 116534187945 | tx_bytes: 116534380820 nid001348: tx_packets: 82642359 | tx_packets: 82643634 nid001349: 2022-04-24A--:--:--.------+--:-- nid001349 #### nid001349 - Job 2269370.cbqs01 HSN0 MLX Counter Post-Job Difference nid001349: multicast: 5953009 | multicast: 5953010 nid001349: port_rcv_data: 289160884211164 | port_rcv_data: 289160884222872 nid001349: port_rcv_packets: 558710399114 | port_rcv_packets: 558710399333 nid001349: port_xmit_data: 302116335711314 | port_xmit_data: 302116335723038 nid001349: port_xmit_packets: 567036676118 | port_xmit_packets: 567036676338 nid001349: rx_bytes: 138620200087 | rx_bytes: 138620445605 nid001349: rx_packets: 106394052 | rx_packets: 106394879 nid001349: tx_bytes: 51991162317 | tx_bytes: 51991348968 nid001349: tx_packets: 70140556 | tx_packets: 70141816 nid001349: unicast_rcv_packets: 558710399114 | unicast_rcv_packets: 558710399333 nid001349: unicast_xmit_packets: 567036676118 | unicast_xmit_packets: 567036676338 ------------------------------------------------------------------ ### Job 2269370.cbqs01 - NFS Statistics for job duration: nid001325: #### 2022-04-24.00 #### nid001325 - Job 2269370.cbqs01 NFS Statistics for job duration (nfsstat) nid001325: ## /usr/sbin/nfsstat -v -S /tmp/nfsstats.begin.2269370.cbqs01 : nid001325: Client packet stats: nid001325: packets udp tcp tcpconn nid001325: 0 0 0 0 nid001325: nid001325: Client rpc stats: nid001325: calls retrans authrefrsh nid001325: 171 0 171 nid001325: nid001325: Client nfs v3: nid001325: null getattr setattr lookup access nid001325: 0 0% 65 38% 0 0% 13 7% 69 40% nid001325: readlink read write create mkdir nid001325: 0 0% 19 11% 1 0% 0 0% 0 0% nid001325: symlink mknod remove rmdir rename nid001325: 0 0% 0 0% 0 0% 0 0% 0 0% nid001325: link readdir readdirplus fsstat fsinfo nid001325: 0 0% 0 0% 4 2% 0 0% 0 0% nid001325: pathconf commit nid001325: 0 0% 0 0% nid001325: nid001325: nid001325: ----------------------------------------------------------------------------------- nid001325: nid001325: #### 2022-04-24.00 #### nid001325 - Job 2269370.cbqs01 NFS Mount Statistics for job duration (mountstats) nid001325: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /apps : nid001325: nid001325: nid001325: 172.20.250.16:/AZ-HFS-Cactus-apps mounted on /apps: nid001325: nid001325: ops/s rpc bklog nid001325: 66.000 0.000 nid001325: nid001325: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001325: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001325: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001325: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001325: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /sfs : nid001325: nid001325: nid001325: 172.20.250.16:/AZ-HFS-Cactus-sfs mounted on /sfs: nid001325: nid001325: ops/s rpc bklog nid001325: 66.000 0.000 nid001325: nid001325: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001325: 0.500 2.027 4.055 0 (0.0%) 0.000 1.000 nid001325: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001325: 0.500 0.812 1.625 0 (0.0%) 1.000 1.000 nid001325: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /u : nid001325: nid001325: nid001325: 172.20.250.17:/AZ-HFS-Cactus-u mounted on /u: nid001325: nid001325: ops/s rpc bklog nid001325: 11.000 0.000 nid001325: nid001325: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001325: 0.500 0.178 0.355 0 (0.0%) 0.000 0.000 nid001325: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001325: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001325: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /pe : nid001325: nid001325: nid001325: 10.31.62.244:/cm_shared/image/images_rw_nfs/pe mounted on /pe: nid001325: nid001325: ops/s rpc bklog nid001325: 66.000 0.000 nid001325: nid001325: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001325: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001325: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001325: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001325: ---------------- nid001325: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /apps : nid001325: Stats for 172.20.250.16:/AZ-HFS-Cactus-apps mounted on /apps: nid001325: NFS mount options: ro,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.20.250.16,mountvers=3,mountport=4048,mountproto=udp,local_lock=none nid001325: NFS server capabilities: caps=0x3fc7,wtmult=4096,dtsize=8192,bsize=0,namlen=255 nid001325: NFS security flavor: 1 pseudoflavor: 0 nid001325: nid001325: NFS byte counts: nid001325: applications read 58671 bytes via read(2) nid001325: applications wrote 0 bytes via write(2) nid001325: applications read 0 bytes via O_DIRECT read(2) nid001325: applications wrote 0 bytes via O_DIRECT write(2) nid001325: client read 0 bytes via NFS READ nid001325: client wrote 0 bytes via NFS WRITE nid001325: nid001325: RPC statistics: nid001325: 132 RPC requests sent, 132 RPC replies received (0 XIDs not found) nid001325: average backlog queue length: 0 nid001325: nid001325: ACCESS: nid001325: 64 ops (48%) nid001325: avg bytes sent per op: avg bytes received per op: 120 nid001325: backlog wait: 0.000000 RTT: 0.187500 total execute time: 0.187500 (milliseconds) nid001325: GETATTR: nid001325: 55 ops (41%) nid001325: avg bytes sent per op: avg bytes received per op: 112 nid001325: backlog wait: 0.000000 RTT: 0.054545 total execute time: 0.072727 (milliseconds) nid001325: READDIRPLUS: nid001325: 2 ops (1%) nid001325: avg bytes sent per op: avg bytes received per op: 606 nid001325: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001325: nid001325: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /sfs : nid001325: nid001325: Stats for 172.20.250.16:/AZ-HFS-Cactus-sfs mounted on /sfs: nid001325: NFS mount options: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.20.250.16,mountvers=3,mountport=4048,mountproto=udp,local_lock=none nid001325: NFS server capabilities: caps=0x3fc7,wtmult=4096,dtsize=8192,bsize=0,namlen=255 nid001325: NFS security flavor: 1 pseudoflavor: 0 nid001325: nid001325: NFS byte counts: nid001325: applications read 13051 bytes via read(2) nid001325: applications wrote 958 bytes via write(2) nid001325: applications read 0 bytes via O_DIRECT read(2) nid001325: applications wrote 0 bytes via O_DIRECT write(2) nid001325: client read 3894 bytes via NFS READ nid001325: client wrote 1367 bytes via NFS WRITE nid001325: nid001325: RPC statistics: nid001325: 132 RPC requests sent, 132 RPC replies received (0 XIDs not found) nid001325: average backlog queue length: 0 nid001325: nid001325: GETATTR: nid001325: 7 ops (5%) nid001325: avg bytes sent per op: avg bytes received per op: 112 nid001325: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.142857 (milliseconds) nid001325: LOOKUP: nid001325: 1 ops (0%) nid001325: avg bytes sent per op: avg bytes received per op: 240 nid001325: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001325: ACCESS: nid001325: 1 ops (0%) nid001325: avg bytes sent per op: avg bytes received per op: 120 nid001325: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001325: READ: nid001325: 1 ops (0%) nid001325: avg bytes sent per op: avg bytes received per op: 4024 nid001325: backlog wait: 0.000000 RTT: 0.000000 total execute time: 1.000000 (milliseconds) nid001325: WRITE: nid001325: 1 ops (0%) nid001325: avg bytes sent per op: avg bytes received per op: 160 nid001325: backlog wait: 0.000000 RTT: 1.000000 total execute time: 1.000000 (milliseconds) nid001325: nid001325: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /u : nid001325: nid001325: Stats for 172.20.250.17:/AZ-HFS-Cactus-u mounted on /u: nid001325: NFS mount options: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.20.250.17,mountvers=3,mountport=4048,mountproto=udp,local_lock=none nid001325: NFS server capabilities: caps=0x3fc7,wtmult=4096,dtsize=8192,bsize=0,namlen=255 nid001325: NFS security flavor: 1 pseudoflavor: 0 nid001325: nid001325: NFS byte counts: nid001325: applications read 70 bytes via read(2) nid001325: applications wrote 0 bytes via write(2) nid001325: applications read 0 bytes via O_DIRECT read(2) nid001325: applications wrote 0 bytes via O_DIRECT write(2) nid001325: client read 70 bytes via NFS READ nid001325: client wrote 0 bytes via NFS WRITE nid001325: nid001325: RPC statistics: nid001325: 22 RPC requests sent, 22 RPC replies received (0 XIDs not found) nid001325: average backlog queue length: 0 nid001325: nid001325: LOOKUP: nid001325: 12 ops (54%) nid001325: avg bytes sent per op: avg bytes received per op: 136 nid001325: backlog wait: 0.000000 RTT: 0.083333 total execute time: 0.083333 (milliseconds) nid001325: ACCESS: nid001325: 4 ops (18%) nid001325: avg bytes sent per op: avg bytes received per op: 120 nid001325: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001325: GETATTR: nid001325: 3 ops (13%) nid001325: avg bytes sent per op: avg bytes received per op: 112 nid001325: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001325: READDIRPLUS: nid001325: 2 ops (9%) nid001325: avg bytes sent per op: avg bytes received per op: 2546 nid001325: backlog wait: 0.000000 RTT: 1.000000 total execute time: 0.500000 (milliseconds) nid001325: READ: nid001325: 1 ops (4%) nid001325: avg bytes sent per op: avg bytes received per op: 200 nid001325: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001325: nid001325: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /pe : nid001325: nid001325: Stats for 10.31.62.244:/cm_shared/image/images_rw_nfs/pe mounted on /pe: nid001325: NFS mount options: ro,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,nolock,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=10.31.62.244,mountvers=3,mountport=38465,mountproto=tcp,local_lock=all nid001325: NFS server capabilities: caps=0x3fcf,wtmult=4096,dtsize=32768,bsize=0,namlen=255 nid001325: NFS security flavor: 1 pseudoflavor: 0 nid001325: nid001325: NFS byte counts: nid001325: applications read 0 bytes via read(2) nid001325: applications wrote 0 bytes via write(2) nid001325: applications read 0 bytes via O_DIRECT read(2) nid001325: applications wrote 0 bytes via O_DIRECT write(2) nid001325: client read 0 bytes via NFS READ nid001325: client wrote 0 bytes via NFS WRITE nid001325: nid001325: RPC statistics: nid001325: 132 RPC requests sent, 132 RPC replies received (0 XIDs not found) nid001325: average backlog queue length: 0 nid001325: nid001325: nid001325: nid001346: #### 2022-04-24.00 #### nid001346 - Job 2269370.cbqs01 NFS Statistics for job duration (nfsstat) nid001346: ## /usr/sbin/nfsstat -v -S /tmp/nfsstats.begin.2269370.cbqs01 : nid001346: Client packet stats: nid001346: packets udp tcp tcpconn nid001346: 0 0 0 0 nid001346: nid001346: Client rpc stats: nid001346: calls retrans authrefrsh nid001346: 219 0 219 nid001346: nid001346: Client nfs v3: nid001346: null getattr setattr lookup access nid001346: 0 0% 217 99% 0 0% 1 0% 1 0% nid001346: readlink read write create mkdir nid001346: 0 0% 0 0% 0 0% 0 0% 0 0% nid001346: symlink mknod remove rmdir rename nid001346: 0 0% 0 0% 0 0% 0 0% 0 0% nid001346: link readdir readdirplus fsstat fsinfo nid001346: 0 0% 0 0% 0 0% 0 0% 0 0% nid001346: pathconf commit nid001346: 0 0% 0 0% nid001346: nid001346: nid001346: ----------------------------------------------------------------------------------- nid001346: nid001346: #### 2022-04-24.00 #### nid001346 - Job 2269370.cbqs01 NFS Mount Statistics for job duration (mountstats) nid001346: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /apps : nid001346: nid001346: nid001346: 172.20.250.16:/AZ-HFS-Cactus-apps mounted on /apps: nid001346: nid001346: ops/s rpc bklog nid001346: 55.333 0.000 nid001346: nid001346: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001346: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001346: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001346: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001346: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /sfs : nid001346: nid001346: nid001346: 172.20.250.16:/AZ-HFS-Cactus-sfs mounted on /sfs: nid001346: nid001346: ops/s rpc bklog nid001346: 55.333 0.000 nid001346: nid001346: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001346: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001346: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001346: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001346: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /u : nid001346: nid001346: nid001346: 172.20.250.17:/AZ-HFS-Cactus-u mounted on /u: nid001346: nid001346: ops/s rpc bklog nid001346: 0.000 0.000 nid001346: nid001346: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001346: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001346: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001346: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001346: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /pe : nid001346: nid001346: nid001346: 10.31.62.244:/cm_shared/image/images_rw_nfs/pe mounted on /pe: nid001346: nid001346: ops/s rpc bklog nid001346: 40.333 0.000 nid001346: nid001346: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001346: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001346: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001346: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001346: ---------------- nid001346: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /apps : nid001346: Stats for 172.20.250.16:/AZ-HFS-Cactus-apps mounted on /apps: nid001346: NFS mount options: ro,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.20.250.16,mountvers=3,mountport=4048,mountproto=udp,local_lock=none nid001346: NFS server capabilities: caps=0x3fc7,wtmult=4096,dtsize=8192,bsize=0,namlen=255 nid001346: NFS security flavor: 1 pseudoflavor: 0 nid001346: nid001346: NFS byte counts: nid001346: applications read 176013 bytes via read(2) nid001346: applications wrote 0 bytes via write(2) nid001346: applications read 0 bytes via O_DIRECT read(2) nid001346: applications wrote 0 bytes via O_DIRECT write(2) nid001346: client read 0 bytes via NFS READ nid001346: client wrote 0 bytes via NFS WRITE nid001346: nid001346: RPC statistics: nid001346: 166 RPC requests sent, 166 RPC replies received (0 XIDs not found) nid001346: average backlog queue length: 0 nid001346: nid001346: GETATTR: nid001346: 159 ops (95%) nid001346: avg bytes sent per op: avg bytes received per op: 112 nid001346: backlog wait: 0.000000 RTT: 0.050314 total execute time: 0.062893 (milliseconds) nid001346: nid001346: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /sfs : nid001346: nid001346: Stats for 172.20.250.16:/AZ-HFS-Cactus-sfs mounted on /sfs: nid001346: NFS mount options: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.20.250.16,mountvers=3,mountport=4048,mountproto=udp,local_lock=none nid001346: NFS server capabilities: caps=0x3fc7,wtmult=4096,dtsize=8192,bsize=0,namlen=255 nid001346: NFS security flavor: 1 pseudoflavor: 0 nid001346: nid001346: NFS byte counts: nid001346: applications read 12189 bytes via read(2) nid001346: applications wrote 131 bytes via write(2) nid001346: applications read 0 bytes via O_DIRECT read(2) nid001346: applications wrote 0 bytes via O_DIRECT write(2) nid001346: client read 0 bytes via NFS READ nid001346: client wrote 0 bytes via NFS WRITE nid001346: nid001346: RPC statistics: nid001346: 166 RPC requests sent, 166 RPC replies received (0 XIDs not found) nid001346: average backlog queue length: 0 nid001346: nid001346: GETATTR: nid001346: 5 ops (3%) nid001346: avg bytes sent per op: avg bytes received per op: 112 nid001346: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.200000 (milliseconds) nid001346: LOOKUP: nid001346: 1 ops (0%) nid001346: avg bytes sent per op: avg bytes received per op: 240 nid001346: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001346: ACCESS: nid001346: 1 ops (0%) nid001346: avg bytes sent per op: avg bytes received per op: 120 nid001346: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001346: nid001346: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /u : nid001346: nid001346: Stats for 172.20.250.17:/AZ-HFS-Cactus-u mounted on /u: nid001346: NFS mount options: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.20.250.17,mountvers=3,mountport=4048,mountproto=udp,local_lock=none nid001346: NFS server capabilities: caps=0x3fc7,wtmult=4096,dtsize=8192,bsize=0,namlen=255 nid001346: NFS security flavor: 1 pseudoflavor: 0 nid001346: nid001346: NFS byte counts: nid001346: applications read 0 bytes via read(2) nid001346: applications wrote 0 bytes via write(2) nid001346: applications read 0 bytes via O_DIRECT read(2) nid001346: applications wrote 0 bytes via O_DIRECT write(2) nid001346: client read 0 bytes via NFS READ nid001346: client wrote 0 bytes via NFS WRITE nid001346: nid001346: RPC statistics: nid001346: 0 RPC requests sent, 0 RPC replies received (0 XIDs not found) nid001346: nid001346: nid001346: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /pe : nid001346: nid001346: Stats for 10.31.62.244:/cm_shared/image/images_rw_nfs/pe mounted on /pe: nid001346: NFS mount options: ro,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,nolock,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=10.31.62.244,mountvers=3,mountport=38465,mountproto=tcp,local_lock=all nid001346: NFS server capabilities: caps=0x3fcf,wtmult=4096,dtsize=32768,bsize=0,namlen=255 nid001346: NFS security flavor: 1 pseudoflavor: 0 nid001346: nid001346: NFS byte counts: nid001346: applications read 0 bytes via read(2) nid001346: applications wrote 0 bytes via write(2) nid001346: applications read 0 bytes via O_DIRECT read(2) nid001346: applications wrote 0 bytes via O_DIRECT write(2) nid001346: client read 0 bytes via NFS READ nid001346: client wrote 0 bytes via NFS WRITE nid001346: nid001346: RPC statistics: nid001346: 121 RPC requests sent, 115 RPC replies received (0 XIDs not found) nid001346: average backlog queue length: 0 nid001346: nid001346: nid001346: nid001347: #### 2022-04-24.00 #### nid001347 - Job 2269370.cbqs01 NFS Statistics for job duration (nfsstat) nid001347: ## /usr/sbin/nfsstat -v -S /tmp/nfsstats.begin.2269370.cbqs01 : nid001347: Client packet stats: nid001347: packets udp tcp tcpconn nid001347: 0 0 0 0 nid001347: nid001347: Client rpc stats: nid001347: calls retrans authrefrsh nid001347: 220 0 220 nid001347: nid001347: Client nfs v3: nid001347: null getattr setattr lookup access nid001347: 0 0% 217 98% 0 0% 1 0% 1 0% nid001347: readlink read write create mkdir nid001347: 0 0% 1 0% 0 0% 0 0% 0 0% nid001347: symlink mknod remove rmdir rename nid001347: 0 0% 0 0% 0 0% 0 0% 0 0% nid001347: link readdir readdirplus fsstat fsinfo nid001347: 0 0% 0 0% 0 0% 0 0% 0 0% nid001347: pathconf commit nid001347: 0 0% 0 0% nid001347: nid001347: nid001347: ----------------------------------------------------------------------------------- nid001347: nid001347: #### 2022-04-24.00 #### nid001347 - Job 2269370.cbqs01 NFS Mount Statistics for job duration (mountstats) nid001347: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /apps : nid001347: nid001347: nid001347: 172.20.250.16:/AZ-HFS-Cactus-apps mounted on /apps: nid001347: nid001347: ops/s rpc bklog nid001347: 55.667 0.000 nid001347: nid001347: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001347: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001347: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001347: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001347: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /sfs : nid001347: nid001347: nid001347: 172.20.250.16:/AZ-HFS-Cactus-sfs mounted on /sfs: nid001347: nid001347: ops/s rpc bklog nid001347: 55.667 0.000 nid001347: nid001347: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001347: 0.333 1.352 4.055 0 (0.0%) 0.000 0.000 nid001347: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001347: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001347: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /u : nid001347: nid001347: nid001347: 172.20.250.17:/AZ-HFS-Cactus-u mounted on /u: nid001347: nid001347: ops/s rpc bklog nid001347: 0.000 0.000 nid001347: nid001347: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001347: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001347: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001347: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001347: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /pe : nid001347: nid001347: nid001347: 10.31.62.244:/cm_shared/image/images_rw_nfs/pe mounted on /pe: nid001347: nid001347: ops/s rpc bklog nid001347: 57.500 0.000 nid001347: nid001347: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001347: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001347: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001347: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001347: ---------------- nid001347: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /apps : nid001347: Stats for 172.20.250.16:/AZ-HFS-Cactus-apps mounted on /apps: nid001347: NFS mount options: ro,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.20.250.16,mountvers=3,mountport=4048,mountproto=udp,local_lock=none nid001347: NFS server capabilities: caps=0x3fc7,wtmult=4096,dtsize=8192,bsize=0,namlen=255 nid001347: NFS security flavor: 1 pseudoflavor: 0 nid001347: nid001347: NFS byte counts: nid001347: applications read 176013 bytes via read(2) nid001347: applications wrote 0 bytes via write(2) nid001347: applications read 0 bytes via O_DIRECT read(2) nid001347: applications wrote 0 bytes via O_DIRECT write(2) nid001347: client read 0 bytes via NFS READ nid001347: client wrote 0 bytes via NFS WRITE nid001347: nid001347: RPC statistics: nid001347: 167 RPC requests sent, 167 RPC replies received (0 XIDs not found) nid001347: average backlog queue length: 0 nid001347: nid001347: GETATTR: nid001347: 159 ops (95%) nid001347: avg bytes sent per op: avg bytes received per op: 112 nid001347: backlog wait: 0.000000 RTT: 0.056604 total execute time: 0.069182 (milliseconds) nid001347: nid001347: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /sfs : nid001347: nid001347: Stats for 172.20.250.16:/AZ-HFS-Cactus-sfs mounted on /sfs: nid001347: NFS mount options: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.20.250.16,mountvers=3,mountport=4048,mountproto=udp,local_lock=none nid001347: NFS server capabilities: caps=0x3fc7,wtmult=4096,dtsize=8192,bsize=0,namlen=255 nid001347: NFS security flavor: 1 pseudoflavor: 0 nid001347: nid001347: NFS byte counts: nid001347: applications read 12189 bytes via read(2) nid001347: applications wrote 131 bytes via write(2) nid001347: applications read 0 bytes via O_DIRECT read(2) nid001347: applications wrote 0 bytes via O_DIRECT write(2) nid001347: client read 3894 bytes via NFS READ nid001347: client wrote 0 bytes via NFS WRITE nid001347: nid001347: RPC statistics: nid001347: 167 RPC requests sent, 167 RPC replies received (0 XIDs not found) nid001347: average backlog queue length: 0 nid001347: nid001347: GETATTR: nid001347: 5 ops (2%) nid001347: avg bytes sent per op: avg bytes received per op: 112 nid001347: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001347: LOOKUP: nid001347: 1 ops (0%) nid001347: avg bytes sent per op: avg bytes received per op: 240 nid001347: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001347: ACCESS: nid001347: 1 ops (0%) nid001347: avg bytes sent per op: avg bytes received per op: 120 nid001347: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001347: READ: nid001347: 1 ops (0%) nid001347: avg bytes sent per op: avg bytes received per op: 4024 nid001347: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001347: nid001347: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /u : nid001347: nid001347: Stats for 172.20.250.17:/AZ-HFS-Cactus-u mounted on /u: nid001347: NFS mount options: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.20.250.17,mountvers=3,mountport=4048,mountproto=udp,local_lock=none nid001347: NFS server capabilities: caps=0x3fc7,wtmult=4096,dtsize=8192,bsize=0,namlen=255 nid001347: NFS security flavor: 1 pseudoflavor: 0 nid001347: nid001347: NFS byte counts: nid001347: applications read 0 bytes via read(2) nid001347: applications wrote 0 bytes via write(2) nid001347: applications read 0 bytes via O_DIRECT read(2) nid001347: applications wrote 0 bytes via O_DIRECT write(2) nid001347: client read 0 bytes via NFS READ nid001347: client wrote 0 bytes via NFS WRITE nid001347: nid001347: RPC statistics: nid001347: 0 RPC requests sent, 0 RPC replies received (0 XIDs not found) nid001347: nid001347: nid001347: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /pe : nid001347: nid001347: Stats for 10.31.62.244:/cm_shared/image/images_rw_nfs/pe mounted on /pe: nid001347: NFS mount options: ro,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,nolock,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=10.31.62.244,mountvers=3,mountport=38465,mountproto=tcp,local_lock=all nid001347: NFS server capabilities: caps=0x3fcf,wtmult=4096,dtsize=32768,bsize=0,namlen=255 nid001347: NFS security flavor: 1 pseudoflavor: 0 nid001347: nid001347: NFS byte counts: nid001347: applications read 0 bytes via read(2) nid001347: applications wrote 0 bytes via write(2) nid001347: applications read 0 bytes via O_DIRECT read(2) nid001347: applications wrote 0 bytes via O_DIRECT write(2) nid001347: client read 0 bytes via NFS READ nid001347: client wrote 0 bytes via NFS WRITE nid001347: nid001347: RPC statistics: nid001347: 115 RPC requests sent, 109 RPC replies received (0 XIDs not found) nid001347: average backlog queue length: 0 nid001347: nid001347: nid001347: nid001348: #### 2022-04-24.00 #### nid001348 - Job 2269370.cbqs01 NFS Statistics for job duration (nfsstat) nid001348: ## /usr/sbin/nfsstat -v -S /tmp/nfsstats.begin.2269370.cbqs01 : nid001348: Client packet stats: nid001348: packets udp tcp tcpconn nid001348: 0 0 0 0 nid001348: nid001348: Client rpc stats: nid001348: calls retrans authrefrsh nid001348: 220 0 220 nid001348: nid001348: Client nfs v3: nid001348: null getattr setattr lookup access nid001348: 0 0% 217 98% 0 0% 1 0% 1 0% nid001348: readlink read write create mkdir nid001348: 0 0% 1 0% 0 0% 0 0% 0 0% nid001348: symlink mknod remove rmdir rename nid001348: 0 0% 0 0% 0 0% 0 0% 0 0% nid001348: link readdir readdirplus fsstat fsinfo nid001348: 0 0% 0 0% 0 0% 0 0% 0 0% nid001348: pathconf commit nid001348: 0 0% 0 0% nid001348: nid001348: nid001348: ----------------------------------------------------------------------------------- nid001348: nid001348: #### 2022-04-24.00 #### nid001348 - Job 2269370.cbqs01 NFS Mount Statistics for job duration (mountstats) nid001348: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /apps : nid001348: nid001348: nid001348: 172.20.250.16:/AZ-HFS-Cactus-apps mounted on /apps: nid001348: nid001348: ops/s rpc bklog nid001348: 83.500 0.000 nid001348: nid001348: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001348: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001348: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001348: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001348: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /sfs : nid001348: nid001348: nid001348: 172.20.250.16:/AZ-HFS-Cactus-sfs mounted on /sfs: nid001348: nid001348: ops/s rpc bklog nid001348: 83.500 0.000 nid001348: nid001348: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001348: 0.500 2.027 4.055 0 (0.0%) 0.000 0.000 nid001348: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001348: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001348: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /u : nid001348: nid001348: nid001348: 172.20.250.17:/AZ-HFS-Cactus-u mounted on /u: nid001348: nid001348: ops/s rpc bklog nid001348: 0.000 0.000 nid001348: nid001348: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001348: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001348: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001348: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001348: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /pe : nid001348: nid001348: nid001348: 10.31.62.244:/cm_shared/image/images_rw_nfs/pe mounted on /pe: nid001348: nid001348: ops/s rpc bklog nid001348: 60.000 0.000 nid001348: nid001348: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001348: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001348: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001348: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001348: ---------------- nid001348: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /apps : nid001348: Stats for 172.20.250.16:/AZ-HFS-Cactus-apps mounted on /apps: nid001348: NFS mount options: ro,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.20.250.16,mountvers=3,mountport=4048,mountproto=udp,local_lock=none nid001348: NFS server capabilities: caps=0x3fc7,wtmult=4096,dtsize=8192,bsize=0,namlen=255 nid001348: NFS security flavor: 1 pseudoflavor: 0 nid001348: nid001348: NFS byte counts: nid001348: applications read 176013 bytes via read(2) nid001348: applications wrote 0 bytes via write(2) nid001348: applications read 0 bytes via O_DIRECT read(2) nid001348: applications wrote 0 bytes via O_DIRECT write(2) nid001348: client read 0 bytes via NFS READ nid001348: client wrote 0 bytes via NFS WRITE nid001348: nid001348: RPC statistics: nid001348: 167 RPC requests sent, 167 RPC replies received (0 XIDs not found) nid001348: average backlog queue length: 0 nid001348: nid001348: GETATTR: nid001348: 159 ops (95%) nid001348: avg bytes sent per op: avg bytes received per op: 112 nid001348: backlog wait: 0.006289 RTT: 0.056604 total execute time: 0.069182 (milliseconds) nid001348: nid001348: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /sfs : nid001348: nid001348: Stats for 172.20.250.16:/AZ-HFS-Cactus-sfs mounted on /sfs: nid001348: NFS mount options: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.20.250.16,mountvers=3,mountport=4048,mountproto=udp,local_lock=none nid001348: NFS server capabilities: caps=0x3fc7,wtmult=4096,dtsize=8192,bsize=0,namlen=255 nid001348: NFS security flavor: 1 pseudoflavor: 0 nid001348: nid001348: NFS byte counts: nid001348: applications read 12189 bytes via read(2) nid001348: applications wrote 131 bytes via write(2) nid001348: applications read 0 bytes via O_DIRECT read(2) nid001348: applications wrote 0 bytes via O_DIRECT write(2) nid001348: client read 3894 bytes via NFS READ nid001348: client wrote 0 bytes via NFS WRITE nid001348: nid001348: RPC statistics: nid001348: 167 RPC requests sent, 167 RPC replies received (0 XIDs not found) nid001348: average backlog queue length: 0 nid001348: nid001348: GETATTR: nid001348: 5 ops (2%) nid001348: avg bytes sent per op: avg bytes received per op: 112 nid001348: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001348: LOOKUP: nid001348: 1 ops (0%) nid001348: avg bytes sent per op: avg bytes received per op: 240 nid001348: backlog wait: 0.000000 RTT: 1.000000 total execute time: 0.000000 (milliseconds) nid001348: ACCESS: nid001348: 1 ops (0%) nid001348: avg bytes sent per op: avg bytes received per op: 120 nid001348: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001348: READ: nid001348: 1 ops (0%) nid001348: avg bytes sent per op: avg bytes received per op: 4024 nid001348: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001348: nid001348: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /u : nid001348: nid001348: Stats for 172.20.250.17:/AZ-HFS-Cactus-u mounted on /u: nid001348: NFS mount options: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.20.250.17,mountvers=3,mountport=4048,mountproto=udp,local_lock=none nid001348: NFS server capabilities: caps=0x3fc7,wtmult=4096,dtsize=8192,bsize=0,namlen=255 nid001348: NFS security flavor: 1 pseudoflavor: 0 nid001348: nid001348: NFS byte counts: nid001348: applications read 0 bytes via read(2) nid001348: applications wrote 0 bytes via write(2) nid001348: applications read 0 bytes via O_DIRECT read(2) nid001348: applications wrote 0 bytes via O_DIRECT write(2) nid001348: client read 0 bytes via NFS READ nid001348: client wrote 0 bytes via NFS WRITE nid001348: nid001348: RPC statistics: nid001348: 0 RPC requests sent, 0 RPC replies received (0 XIDs not found) nid001348: nid001348: nid001348: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /pe : nid001348: nid001348: Stats for 10.31.62.244:/cm_shared/image/images_rw_nfs/pe mounted on /pe: nid001348: NFS mount options: ro,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,nolock,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=10.31.62.244,mountvers=3,mountport=38465,mountproto=tcp,local_lock=all nid001348: NFS server capabilities: caps=0x3fcf,wtmult=4096,dtsize=32768,bsize=0,namlen=255 nid001348: NFS security flavor: 1 pseudoflavor: 0 nid001348: nid001348: NFS byte counts: nid001348: applications read 0 bytes via read(2) nid001348: applications wrote 0 bytes via write(2) nid001348: applications read 0 bytes via O_DIRECT read(2) nid001348: applications wrote 0 bytes via O_DIRECT write(2) nid001348: client read 0 bytes via NFS READ nid001348: client wrote 0 bytes via NFS WRITE nid001348: nid001348: RPC statistics: nid001348: 120 RPC requests sent, 120 RPC replies received (0 XIDs not found) nid001348: average backlog queue length: 0 nid001348: nid001348: nid001348: nid001349: #### 2022-04-24.00 #### nid001349 - Job 2269370.cbqs01 NFS Statistics for job duration (nfsstat) nid001349: ## /usr/sbin/nfsstat -v -S /tmp/nfsstats.begin.2269370.cbqs01 : nid001349: Client packet stats: nid001349: packets udp tcp tcpconn nid001349: 0 0 0 0 nid001349: nid001349: Client rpc stats: nid001349: calls retrans authrefrsh nid001349: 220 0 220 nid001349: nid001349: Client nfs v3: nid001349: null getattr setattr lookup access nid001349: 0 0% 217 98% 0 0% 1 0% 1 0% nid001349: readlink read write create mkdir nid001349: 0 0% 1 0% 0 0% 0 0% 0 0% nid001349: symlink mknod remove rmdir rename nid001349: 0 0% 0 0% 0 0% 0 0% 0 0% nid001349: link readdir readdirplus fsstat fsinfo nid001349: 0 0% 0 0% 0 0% 0 0% 0 0% nid001349: pathconf commit nid001349: 0 0% 0 0% nid001349: nid001349: nid001349: ----------------------------------------------------------------------------------- nid001349: nid001349: #### 2022-04-24.00 #### nid001349 - Job 2269370.cbqs01 NFS Mount Statistics for job duration (mountstats) nid001349: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /apps : nid001349: nid001349: nid001349: 172.20.250.16:/AZ-HFS-Cactus-apps mounted on /apps: nid001349: nid001349: ops/s rpc bklog nid001349: 83.500 0.000 nid001349: nid001349: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001349: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001349: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001349: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001349: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /sfs : nid001349: nid001349: nid001349: 172.20.250.16:/AZ-HFS-Cactus-sfs mounted on /sfs: nid001349: nid001349: ops/s rpc bklog nid001349: 55.667 0.000 nid001349: nid001349: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001349: 0.333 1.352 4.055 0 (0.0%) 0.000 0.000 nid001349: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001349: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001349: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /u : nid001349: nid001349: nid001349: 172.20.250.17:/AZ-HFS-Cactus-u mounted on /u: nid001349: nid001349: ops/s rpc bklog nid001349: 0.000 0.000 nid001349: nid001349: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001349: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001349: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001349: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001349: ## /usr/sbin/mountstats iostat -S /tmp/mntstats.begin.2269370.cbqs01 /pe : nid001349: nid001349: nid001349: 10.31.62.244:/cm_shared/image/images_rw_nfs/pe mounted on /pe: nid001349: nid001349: ops/s rpc bklog nid001349: 33.333 0.000 nid001349: nid001349: read: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001349: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001349: write: ops/s kB/s kB/op retrans avg RTT (ms) avg exe (ms) nid001349: 0.000 0.000 0.000 0 (0.0%) 0.000 0.000 nid001349: ---------------- nid001349: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /apps : nid001349: Stats for 172.20.250.16:/AZ-HFS-Cactus-apps mounted on /apps: nid001349: NFS mount options: ro,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.20.250.16,mountvers=3,mountport=4048,mountproto=udp,local_lock=none nid001349: NFS server capabilities: caps=0x3fc7,wtmult=4096,dtsize=8192,bsize=0,namlen=255 nid001349: NFS security flavor: 1 pseudoflavor: 0 nid001349: nid001349: NFS byte counts: nid001349: applications read 176013 bytes via read(2) nid001349: applications wrote 0 bytes via write(2) nid001349: applications read 0 bytes via O_DIRECT read(2) nid001349: applications wrote 0 bytes via O_DIRECT write(2) nid001349: client read 0 bytes via NFS READ nid001349: client wrote 0 bytes via NFS WRITE nid001349: nid001349: RPC statistics: nid001349: 167 RPC requests sent, 167 RPC replies received (0 XIDs not found) nid001349: average backlog queue length: 0 nid001349: nid001349: GETATTR: nid001349: 159 ops (95%) nid001349: avg bytes sent per op: avg bytes received per op: 112 nid001349: backlog wait: 0.000000 RTT: 0.062893 total execute time: 0.069182 (milliseconds) nid001349: nid001349: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /sfs : nid001349: nid001349: Stats for 172.20.250.16:/AZ-HFS-Cactus-sfs mounted on /sfs: nid001349: NFS mount options: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.20.250.16,mountvers=3,mountport=4048,mountproto=udp,local_lock=none nid001349: NFS server capabilities: caps=0x3fc7,wtmult=4096,dtsize=8192,bsize=0,namlen=255 nid001349: NFS security flavor: 1 pseudoflavor: 0 nid001349: nid001349: NFS byte counts: nid001349: applications read 12189 bytes via read(2) nid001349: applications wrote 0 bytes via write(2) nid001349: applications read 0 bytes via O_DIRECT read(2) nid001349: applications wrote 0 bytes via O_DIRECT write(2) nid001349: client read 3894 bytes via NFS READ nid001349: client wrote 0 bytes via NFS WRITE nid001349: nid001349: RPC statistics: nid001349: 167 RPC requests sent, 167 RPC replies received (0 XIDs not found) nid001349: average backlog queue length: 0 nid001349: nid001349: GETATTR: nid001349: 5 ops (2%) nid001349: avg bytes sent per op: avg bytes received per op: 112 nid001349: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.200000 (milliseconds) nid001349: LOOKUP: nid001349: 1 ops (0%) nid001349: avg bytes sent per op: avg bytes received per op: 240 nid001349: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001349: ACCESS: nid001349: 1 ops (0%) nid001349: avg bytes sent per op: avg bytes received per op: 120 nid001349: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001349: READ: nid001349: 1 ops (0%) nid001349: avg bytes sent per op: avg bytes received per op: 4024 nid001349: backlog wait: 0.000000 RTT: 0.000000 total execute time: 0.000000 (milliseconds) nid001349: nid001349: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /u : nid001349: nid001349: Stats for 172.20.250.17:/AZ-HFS-Cactus-u mounted on /u: nid001349: NFS mount options: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.20.250.17,mountvers=3,mountport=4048,mountproto=udp,local_lock=none nid001349: NFS server capabilities: caps=0x3fc7,wtmult=4096,dtsize=8192,bsize=0,namlen=255 nid001349: NFS security flavor: 1 pseudoflavor: 0 nid001349: nid001349: NFS byte counts: nid001349: applications read 0 bytes via read(2) nid001349: applications wrote 0 bytes via write(2) nid001349: applications read 0 bytes via O_DIRECT read(2) nid001349: applications wrote 0 bytes via O_DIRECT write(2) nid001349: client read 0 bytes via NFS READ nid001349: client wrote 0 bytes via NFS WRITE nid001349: nid001349: RPC statistics: nid001349: 0 RPC requests sent, 0 RPC replies received (0 XIDs not found) nid001349: nid001349: nid001349: ## /usr/sbin/mountstats mountstats -S /tmp/mntstats.begin.2269370.cbqs01 /pe : nid001349: nid001349: Stats for 10.31.62.244:/cm_shared/image/images_rw_nfs/pe mounted on /pe: nid001349: NFS mount options: ro,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,nolock,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=10.31.62.244,mountvers=3,mountport=38465,mountproto=tcp,local_lock=all nid001349: NFS server capabilities: caps=0x3fcf,wtmult=4096,dtsize=32768,bsize=0,namlen=255 nid001349: NFS security flavor: 1 pseudoflavor: 0 nid001349: nid001349: NFS byte counts: nid001349: applications read 0 bytes via read(2) nid001349: applications wrote 0 bytes via write(2) nid001349: applications read 0 bytes via O_DIRECT read(2) nid001349: applications wrote 0 bytes via O_DIRECT write(2) nid001349: client read 0 bytes via NFS READ nid001349: client wrote 0 bytes via NFS WRITE nid001349: nid001349: RPC statistics: nid001349: 100 RPC requests sent, 100 RPC replies received (0 XIDs not found) nid001349: average backlog queue length: 0 nid001349: nid001349: nid001349: ------------------------------------------------------------------ ### Job 2269370.cbqs01 - Exit status is 1 ------------------------------------------------------------------ Job 2269370.cbqs01 - Job summary: Job Id: 2269370.cbqs01 Job_Name = cpnmmb_2021082406_hmon2 Job_Owner = ops.prod@clogin08.cactus.wcoss2.ncep.noaa.gov resources_used.cpupercent = 25 resources_used.cput = 00:00:02 resources_used.mem = 0kb resources_used.ncpus = 640 resources_used.vmem = 0kb resources_used.walltime = 00:00:00 job_state = R queue = prod server = cbqs01 Account_Name = HMON-OPS Checkpoint = u ctime = Sun Apr 24 16:17:05 2022 Error_Path = clogin08.cactus.wcoss2.ncep.noaa.gov:/lfs/h1/ops/prod/packages /hmon.v3.2.0/jobs/cpnmmb_2021082406_hmon2.e2269370 exec_host = nid001325/0*128+nid001346/0*128+nid001347/0*128+nid001348/0*128 +nid001349/0*128 exec_vnode = (nid001325:ncpus=128)+(nid001346:ncpus=128)+(nid001347:ncpus=1 28)+(nid001348:ncpus=128)+(nid001349:ncpus=128) Hold_Types = n Join_Path = oe Keep_Files = oed Mail_Points = a mtime = Sun Apr 24 16:17:13 2022 Output_Path = clogin08.cactus.wcoss2.ncep.noaa.gov:/lfs/h1/ops/prod/package s/hmon.v3.2.0/jobs/cpnmmb_2021082406_hmon2.o2269370 Priority = 0 qtime = Sun Apr 24 16:17:05 2022 Rerunable = True Resource_List.alvl = 2 Resource_List.aslr = True Resource_List.debug = True Resource_List.dfs = False Resource_List.hyper = False Resource_List.mpiprocs = 635 Resource_List.ncpus = 640 Resource_List.nodect = 5 Resource_List.place = excl Resource_List.select = 5:ncpus=128:mpiprocs=127 Resource_List.thp = True Resource_List.turbo = True Resource_List.walltime = 02:30:00 schedselect = 5:ncpus=128:mpiprocs=127:prepost=False stime = Sun Apr 24 16:17:05 2022 session_id = 136783 jobdir = /u/ops.prod substate = 42 Variable_List = PBS_O_HOME=/u/ops.prod,PBS_O_LANG=en_US.UTF-8, PBS_O_LOGNAME=ops.prod, PBS_O_PATH=/opt/cray/libfabric/1.11.0.0./bin:/opt/clmgr/sbin:/opt/clmg r/bin:/opt/sgi/sbin:/opt/sgi/bin:/usr/local/bin:/usr/bin:/bin:/usr/lib/ mit/bin:/usr/lib/mit/sbin:/opt/pbs/bin:/sbin, PBS_O_MAIL=/var/mail/ops.prod,PBS_O_SHELL=/bin/bash, PBS_O_WORKDIR=/lfs/h1/ops/prod/packages/hmon.v3.2.0/jobs, PBS_O_SYSTEM=Linux,PBS_O_QUEUE=prod, PBS_O_HOST=clogin08.cactus.wcoss2.ncep.noaa.gov euser = ops.prod egroup = prod hashname = 2269370.cbqs01 queue_rank = 1650817025331 queue_type = E comment = Job run at Sun Apr 24 at 16:17 on (nid001325:ncpus=128)+(nid00134 6:ncpus=128)+(nid001347:ncpus=128)+(nid001348:ncpus=128)+(nid001349:ncp us=128) etime = Sun Apr 24 16:17:05 2022 umask = 22 run_count = 1 eligible_time = 00:00:00 accrue_type = 3 Submit_arguments = JHMON_FORECAST_2021082406_hmon2 project = HMON-OPS run_version = 1 Submit_Host = clogin08.cactus.wcoss2.ncep.noaa.gov ------------------------------------------------------------------ Job 2269370.cbqs01 - PBS tracejob output (for parent mom node only): Job: 2269370.cbqs01 04/24/2022 16:17:05 M nprocs: 2809, cantstat: 0, nomem: 0, skipped: 2740, cached: 0 04/24/2022 16:17:05 M running prologue 04/24/2022 16:17:13 M Started, pid = 136783 04/24/2022 16:17:13 M task 00000001 terminated 04/24/2022 16:17:13 M Terminated 04/24/2022 16:17:13 M task 00000001 cput=00:00:02 04/24/2022 16:17:13 M kill_job 04/24/2022 16:17:13 M nid001325 cput=00:00:02 mem=852kb 04/24/2022 16:17:13 M nid001346.cactus.wcoss2.ncep.noaa.gov cput=00:00:00 mem=0kb 04/24/2022 16:17:13 M nid001347.cactus.wcoss2.ncep.noaa.gov cput=00:00:00 mem=0kb 04/24/2022 16:17:13 M nid001348.cactus.wcoss2.ncep.noaa.gov cput=00:00:00 mem=0kb 04/24/2022 16:17:13 M nid001349.cactus.wcoss2.ncep.noaa.gov cput=00:00:00 mem=0kb 04/24/2022 16:17:13 M running epilogue ------------------------------------------------------------------ To see full PBS log data, run: /sfs/admin/scripts/tracejob.sh 2269370.cbqs01 ================================================================== END - DEBUG INFO ================================================================== ##### Job 2269370.cbqs01 - PBS Job Script: #! /bin/sh # JHMON_FORECAST - this job runs the 126hr HMON forecast, either # coupled or uncoupled. The coupled forecast requires 5 more # compute nodes (-n 640) than the uncoupled (-n 512). When the coupled # forecast failed, it will fall back to run uncoupled forecast. # # Runs in parallel with: # JHMON_POST1 # JHMON_POST2 # JHMON_HYCOM_POST (coupled only) # # Dependencies: # JHMON_HYCOM_INIT1 (coupled only) # JHMON_HYCOM_INIT2 (coupled only) # JHMON_NPS_BC # JHMON_RELOCATE # # Next Job: (above post jobs run in parallel) # #PBS -N cpnmmb_2021082406_hmon2 #PBS -N cpnmmb_2021082406_hmon2 #PBS -A HMON-OPS #PBS -A HMON-OPS #PBS -q prod #PBS -q prod #PBS -l place=excl,select=5:ncpus=128:mpiprocs=127 #PBS -l place=excl,select=5:ncpus=128:mpiprocs=127 #PBS -l debug=true #PBS -l debug=true #PBS -l walltime=02:30:00 #PBS -l walltime=02:30:00 #PBS -j oe #PBS -j oe date #export PS4='+ $SECONDS + ' set -xu #################################### # Set default values if not yet set # values from ecFlow or modules will override these #################################### #export DATA=${DATA:-${DATAROOT:?}/hmon${storm_num:?}_${cyc:?}_${envir:?}} #export DATA=/lfs/h2/emc/ptmp/Lin.L.Zhu/hmon.v3.2.0/2021082406/13E export DATA=/lfs/f1/ops/prod/tmp/hmon2_06_prod cyc=06 export cycle=${cycle:-t${cyc}z} export jlogfile=${DATA}/jlogfile #################################### # Initialize work area and cd there #################################### mkdir -p $DATA cd $DATA #################################### # Variables for logging & DBnet #################################### export NET=hmon RUN=hmon #################################### # Set threading defaults #################################### export OMP_NUM_THREADS=1 export OMP_STACKSIZE=128M export MKL_NUM_THREADS=1 #################################### # Set HMON-specific vars #################################### # Top of HMON installation tree: #export HOMEhmon=${HOMEhmon:-${PACKAGEROOT:?}/hmon.${hmon_ver:?}} export HOMEhmon=/lfs/h1/ops/prod/packages/hmon.v3.2.0 export USHhmon=$HOMEhmon/ush export EXhmon=$HOMEhmon/scripts export PARMhmon=$HOMEhmon/parm export FIXhmon=$HOMEhmon/fix export EXEChmon=$HOMEhmon/exec export WORKhmon=$DATA source ${HOMEhmon}/versions/run.ver module load PrgEnv-intel/${PrgEnv_intel_ver} module load intel/${intel_ver} module load craype/${craype_ver} module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} #module load iobuf/2.0.10 module load libjpeg/${libjpeg_ver} module load grib_util/${grib_util_ver} module load wgrib2/${wgrib2_ver} module load python/${python_ver} module load cfp/${cfp_ver} module load atp/${atp_ver} module load hdf5/${hdf5_ver} module load netcdf/${netcdf_ver} module load python/${python_ver} # Get PDY #. ./PDY # set up cycle #export CYCLE=${PDY}${cyc} export CYCLE=2021082406 storm_num=2 # Upstream and downstream areas: #export stormlabel=${stormlabel:-storm$storm_num} #export COMINmsg=${COMINmsg:-$(compath.py hwrf/${hwrf_ver}/inphmon)} # HMON areas: #priorymdh=$( $NDATE -6 "$PDY$cyc" ) priorymdh=2021082400 #export HISTDATA=${HISTDATA:-$(compath.py $NET/${hmon_ver})/$RUN.$priorymdh} export HISTDATA=/lfs/h1/ops/canned/com/hmon/v3.2/hmon.2021082400 #export COMIN=${COMIN:-$(compath.py $NET/${hmon_ver})/$RUN.$PDY$cyc} export COMIN=/lfs/h1/ops/prod/com/hmon/v3.2/hmon.2021082406 #export COMOUT=${COMOUT:-$(compath.py -o $NET/${hmon_ver})/$RUN.$PDY$cyc} export COMOUT=/lfs/h1/ops/prod/com/hmon/v3.2/hmon.2021082406 #export COM=${COM:-${COMOUT}} export COM=/lfs/h1/ops/prod/com/hmon/v3.2/hmon.2021082406 # Prepend $USHhmon to python package search path: export PYTHONPATH=$USHhmon${PYTHONPATH:+:$PYTHONPATH} #messagefile=$COMINmsg/message$storm_num #message=$( head -1 "$messagefile" ) # full message contents #basin1=${message:7:1} # L # Make sure all mandatory variables are set: cyc, storm_num set -u +x echo "Checking for mandatory variables:" echo "cyc=$cyc storm_num=$storm_num" set +u -x env #################################### # Set/check job-specific vars if any #################################### #################################### # Pass control to ex-script #################################### #echo "$FORECAST_CONF_OVERRIDE" > "$COMOUT/storm$storm_num.forecast_override" #postmsg "$jlogfile" "Starting exhmon_forecast.py $FORECAST_CONF_OVERRIDE" #$EXhmon/exhmon_nmmb.py $FORECAST_CONF_OVERRIDE # JY - if [[ ! ( "$storm_num" -le "$nstorms" ) ]] ; then #if [[ "$basin1" == "L" ]]; then # export TOTAL_TASKS="412" # $EXhmon/exhmon_nmmb.py $TOTAL_TASKS if [ `echo $job |grep noncouple` ]; then export TOTAL_TASKS=${TOTAL_TASKS:-"412"} $EXhmon/exhmon_nmmb.py $TOTAL_TASKS else export TOTAL_TASKS=${TOTAL_TASKS:-"624"} $EXhmon/exhmon_cpl.py $TOTAL_TASKS fi export err=$?; #err_chk postmsg "$jlogfile" "exhmon_forecast.py ended normally." ##### End of job script ------------------------------------------------------------------