# WCOSS PLATFORMS

wcoss.common={   # Common resources for all WCOSSes
    use platform.defaults
    MACHINE_ID='wcoss'

    HOMEnems=PWD_UP5
    HOMErt=OUTPUT_PATH
    INPUTS="@[BASELINE]"
#    INPUTS='/meso/noscrub/wx20rv/REGRESSION_TEST'
}

platform wcoss.phase2 {
    use wcoss.common
    CPU_ACCOUNT='GFS-T2O'
    pex='1'
    cores_per_node=24
    cpus_per_core=2
    C768_PPN=6
    C768_THRD=1

    # WCOSS Phase 2 has been slow of late.
    DEFAULT_TEST_WALLTIME=3000

    MPI='LSF'
    SHORT_TEST_QUEUE='&SHORTQ;'
    LONG_TEST_QUEUE='&LONGQ;'
    BUILD_QUEUE='&BUILDQ;'
    MACHINE_ID='wcoss.phase2'
    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200210"
    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200210"

    default_resources={
        TASKS=156
        WRTTASK_PER_GROUP=12
        TPN=12
        INPES=3
        JNPES=8
        THRD=1
        atm_petlist_bounds="0 155"
        wav_petlist_bounds="156 263"
        CPLTASKS=264
    }
    default_stretched_resources={
        TASKS=48
        TPN=12
        INPES=2
        JNPES=4
        THRD=2
    }
    default_nested_resources={
	use default_stretched_resources
        TASKS=96
    }
    default_threaded_resources={
        TASKS=84
        WRTTASK_PER_GROUP=12
        TPN=6
        INPES=3
        JNPES=4
        THRD=2
    }

    embed bash detect [[[
        # This function is used at PARSE TIME to detect whether we are
        # on WCOSS Phase 2.  It must be very fast and low resource
        # usage since the parser runs it.
        if [[ -d /usrx && -d /global && -e /etc/redhat-release && \
              -e /etc/prod ]] ; then
            # We are on WCOSS Phase 1 or 2.
            if ( cat /proc/cpuinfo |grep 'processor.*32' > /dev/null ) ; then
                # Not 32 fake (hyperthreading) cpus, so Phase 2.
                exit 0
            fi
        fi
        exit 1
    ]]]

    # ROCOTO SUPPORT ---------------------------------------------------

    # You MUST reduce the TASK_THROTTLE to 1 if you use a debug queue
    TASK_THROTTLE='11'

    rocoto={
        install_dir="@[HOMErt]"
        log_dir="@[TMPrt]/log"
        scheduler="lsf"
        entities=[[[
  <!-- You MUST reduce the TASK_THROTTLE to 1 if you use a debug queue -->
  <!ENTITY SHORTQ "dev2">
  <!ENTITY LONGQ "dev2">
  <!ENTITY BUILDQ "dev2_shared">
]]]
        build_resources=[[[
      <!-- Build job resources minus wallclock -->
      <queue>@[BUILD_QUEUE]</queue>
      <memory>4G</memory>
      <cores>1</cores>
      <native>-R 'affinity[core(1)]'</native>
]]]
        short_test_resources=[[[
    <!-- Short test job -->
    <queue>@[SHORT_TEST_QUEUE]</queue>
    <memory></memory> <!-- Unlimited memory -->
    <native>-a poe</native>
]]]
        long_test_resources=[[[
    <!-- Long test job -->
    <queue>@[LONG_TEST_QUEUE]</queue>
    <memory></memory> <!-- Unlimited memory -->
    <native>-a poe</native>
]]]
    }

}

platform wcoss.phase1 {
    # Implement phase 1 support as a trivial extension of phase 2
    # 16 cores per node instead of 24, use dev queue instead of dev2
    use wcoss.phase2
    pex='1'
    cores_per_node=16
    cpus_per_core=2
    C768_PPN=6
    C768_THRD=4

    MACHINE_ID='wcoss.phase1'
    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200210"
    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200210"

    # WCOSS Phase 2 has been slow of late.
    DEFAULT_TEST_WALLTIME=2700

    # WCOSS Phase 2 has been slow of late.
    DEFAULT_TEST_WALLTIME=2700

    default_resources={
        TASKS=156
        WRTTASK_PER_GROUP=12
        TPN=8
        INPES=3
        JNPES=8
        THRD=1
        atm_petlist_bounds="0 155"
        wav_petlist_bounds="156 263"
        CPLTASKS=264
    }
    default_stretched_resources={
        TASKS=48
        TPN=8
        INPES=2
        JNPES=4
        THRD=2
    }
    default_nested_resources={
	use default_stretched_resources
        TASKS=96
    }
    default_threaded_resources={
        TASKS=84
        WRTTASK_PER_GROUP=12
        TPN=4
        INPES=3
        JNPES=4
        THRD=2
    }

    embed bash detect [[[
        # This function is used at PARSE TIME to detect whether we are
        # on WCOSS Phase 2.  It must be very fast and low resource
        # usage since the parser runs it.
        if [[ -d /usrx && -d /global && -e /etc/redhat-release && \
              -e /etc/prod ]] ; then
            # We are on WCOSS Phase 1 or 2.
            if ( cat /proc/cpuinfo |grep 'processor.*32' > /dev/null ) ; then
                # Not 32 fake (hyperthreading) cpus, so Phase 2.
                exit 1
            fi
            exit 0
        fi
	exit 1
    ]]]

    # ROCOTO SUPPORT ---------------------------------------------------
    rocoto={
        use wcoss.phase2%rocoto
        entities=[[[
  <!ENTITY SHORTQ "dev">
  <!ENTITY LONGQ "dev">
  <!ENTITY BUILDQ "dev_shared">
]]]
    }

}

platform wcoss_dell_p3 {
    # Implement phase 1 support as a trivial extension of phase 2
    # 16 cores per node instead of 24, use dev queue instead of dev2
    use wcoss.common
    CPU_ACCOUNT='GFS-T2O'
    pex='3'
    cores_per_node=28
    cpus_per_core=2
    C768_PPN=7
    C768_THRD=4

    DEFAULT_TEST_WALLTIME=3000

    MPI='LSF_IMPI'
    MACHINE_ID='wcoss_dell_p3'
    SHORT_TEST_QUEUE='&SHORTQ;'
    LONG_TEST_QUEUE='&LONGQ;'
    BUILD_QUEUE='&BUILDQ;'

    BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200210/"
    BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200210/"

    default_resources={
        TASKS=156
        WRTTASK_PER_GROUP=12
        TPN=6
        INPES=3
        JNPES=8
        THRD=4
        atm_petlist_bounds="0 155"
        wav_petlist_bounds="156 263"
        CPLTASKS=264
    }
    default_stretched_resources={
        TASKS=48
        TPN=8
        INPES=2
        JNPES=4
        THRD=2
    }
    default_nested_resources={
	use default_stretched_resources
        TASKS=96
    }
    default_threaded_resources={
        TASKS=84
        WRTTASK_PER_GROUP=12
        TPN=4
        INPES=3
        JNPES=4
        THRD=2
    }

    embed bash detect [[[
        # This function is used at PARSE TIME to detect whether we are
        # on WCOSS Phase 2.  It must be very fast and low resource
        # usage since the parser runs it.
        if [[ -d /usrx ]] ; then
            if ( readlink /usrx | grep dell 2>&1 ) ; then
                exit 0
            fi
        fi
	exit 1
    ]]]


    # ROCOTO SUPPORT ---------------------------------------------------

    # You MUST reduce the TASK_THROTTLE to 1 if you use a debug queue
    TASK_THROTTLE='11'

    rocoto={
        install_dir="@[HOMErt]"
        log_dir="@[TMPrt]/log"
        scheduler="lsf"
        entities=[[[
  <!-- You MUST reduce the TASK_THROTTLE to 1 if you use a debug queue -->
  <!ENTITY SHORTQ "dev">
  <!ENTITY LONGQ "dev">
  <!ENTITY BUILDQ "dev_transfer">
]]]
        build_resources=[[[
      <!-- Build job resources minus wallclock -->
      <queue>@[BUILD_QUEUE]</queue>
      <memory>6G</memory>
      <cores>1</cores>
      <native>-R 'affinity[core(1)]'</native>
]]]
        short_test_resources=[[[
    <!-- Short test job -->
    <queue>@[SHORT_TEST_QUEUE]</queue>
    <memory></memory> <!-- Unlimited memory -->
]]]
        long_test_resources=[[[
    <!-- Long test job -->
    <queue>@[LONG_TEST_QUEUE]</queue>
    <memory></memory> <!-- Unlimited memory -->
]]]
    }

}

platform wcoss.cray {
    use wcoss.common
    MACHINE_ID='wcoss.cray'
    pex='cray'
    cores_per_node=24
    cpus_per_core=2
    C768_PPN=6
    C768_THRD=4

    MPI='lsf_cray_intel'
    SHORT_TEST_QUEUE='&SHORTQ;'
    LONG_TEST_QUEUE='&LONGQ;'
    BUILD_QUEUE='&BUILDQ;'

    TASK_THROTTLE='11'
    BUILD_WALLTIME="3600"
    DEFAULT_TEST_WALLTIME=1800

    BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200210"
    BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200210"

    execution_time_modules=[[[
      module load alps
    ]]]

    default_resources={
        TASKS=150
        WRTTASK_PER_GROUP=6
        TPN=24
        INPES=3
        JNPES=8
        THRD=1
        atm_petlist_bounds="0 149"
        wav_petlist_bounds="150 263"
        CPLTASKS=264
    }
    default_stretched_resources={
        TASKS=48
        TPN=12
        INPES=2
        JNPES=4
        THRD=2
    }
    default_nested_resources={
	use default_stretched_resources
        TASKS=96
    }
    default_threaded_resources={
        TASKS=84
        WRTTASK_PER_GROUP=12
        TPN=12
        INPES=3
        JNPES=4
        THRD=2
    }

    embed bash detect [[[
        # This function is used at PARSE TIME to detect whether we are
        # on WCOSS Cray.  It must be very fast and low resource usage
        # since the parser runs it.
        if [[ -d /gpfs/hps && -e /etc/SuSE-release && -d /usrx ]] ; then
            exit 0
        fi
        exit 1
    ]]]

    # ROCOTO SUPPORT ---------------------------------------------------
    rocoto={
        install_dir="@[HOMErt]"
        log_dir="@[TMPrt]/log"
        scheduler="lsfcray"
        entities=[[[
  <!ENTITY SHORTQ "dev">
  <!ENTITY LONGQ "dev">
  <!ENTITY BUILDQ "dev">
]]]
        build_resources=[[[
      <!-- Build job resources minus wallclock -->
      <queue>@[BUILD_QUEUE]</queue>
      <cores>24</cores>
      <exclusive></exclusive>
]]]
        short_test_resources=[[[
    <!-- Short test job -->
    <queue>@[SHORT_TEST_QUEUE]</queue>
    <exclusive></exclusive>
]]]
        long_test_resources=[[[
    <!-- Long test job -->
    <queue>@[LONG_TEST_QUEUE]</queue>
    <exclusive></exclusive>
]]]
    }
}