This file is indexed.

/usr/bin/ncclimo is in nco 4.6.3-1+b1.

This file is owned by root:root, with mode 0o755.

The actual contents of the file can be viewed below.

   1
   2
   3
   4
   5
   6
   7
   8
   9
  10
  11
  12
  13
  14
  15
  16
  17
  18
  19
  20
  21
  22
  23
  24
  25
  26
  27
  28
  29
  30
  31
  32
  33
  34
  35
  36
  37
  38
  39
  40
  41
  42
  43
  44
  45
  46
  47
  48
  49
  50
  51
  52
  53
  54
  55
  56
  57
  58
  59
  60
  61
  62
  63
  64
  65
  66
  67
  68
  69
  70
  71
  72
  73
  74
  75
  76
  77
  78
  79
  80
  81
  82
  83
  84
  85
  86
  87
  88
  89
  90
  91
  92
  93
  94
  95
  96
  97
  98
  99
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 112
 113
 114
 115
 116
 117
 118
 119
 120
 121
 122
 123
 124
 125
 126
 127
 128
 129
 130
 131
 132
 133
 134
 135
 136
 137
 138
 139
 140
 141
 142
 143
 144
 145
 146
 147
 148
 149
 150
 151
 152
 153
 154
 155
 156
 157
 158
 159
 160
 161
 162
 163
 164
 165
 166
 167
 168
 169
 170
 171
 172
 173
 174
 175
 176
 177
 178
 179
 180
 181
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
#!/bin/bash

# Purpose: Climatology script tailored to CESM'ish monthly input and ACME output guidelines
# Produces and optionally regrids climatological monthly means, seasonal means, annual mean

# Copyright (C) 2015-2016 Charlie Zender
# This file is part of NCO, the netCDF Operators. NCO is free software.
# You may redistribute and/or modify NCO under the terms of the 
# GNU General Public License (GPL) Version 3.

# As a special exception to the terms of the GPL, you are permitted 
# to link the NCO source code with the HDF, netCDF, OPeNDAP, and UDUnits
# libraries and to distribute the resulting executables under the terms 
# of the GPL, but in addition obeying the extra stipulations of the 
# HDF, netCDF, OPeNDAP, and UDUnits licenses.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  
# See the GNU General Public License for more details.

# The original author of this software, Charlie Zender, seeks to improve
# it with your suggestions, contributions, bug-reports, and patches.
# Please contact the NCO project at http://nco.sf.net or write to
# Charlie Zender
# Department of Earth System Science
# University of California, Irvine
# Irvine, CA 92697-3100

# Prerequisites: Bash, NCO
# Script could use other shells, e.g., dash (Debian default) after rewriting function definition and looping constructs

# Source: https://github.com/nco/nco/tree/master/data/ncclimo
# Documentation: http://nco.sf.net/nco.html#ncclimo
# Additional Documentation:
# HowTo: https://acme-climate.atlassian.net/wiki/display/ATM/Generating+and+Regridding+Climatologies+%28climo+files%29+with+NCO+and+ncclimo
# ACME Climatology Requirements: https://acme-climate.atlassian.net/wiki/display/ATM/Climo+Files+-+v0.3+AMIP+runs

# Direct install:
# scp ~/nco/data/ncclimo aims4.llnl.gov:bin
# scp ~/nco/data/ncclimo blues.lcrc.anl.gov:bin
# scp ~/nco/data/ncclimo cooley.alcf.anl.gov:bin
# scp ~/nco/data/ncclimo cori.nersc.gov:bin_cori
# scp ~/nco/data/ncclimo edison.nersc.gov:bin_edison
# scp ~/nco/data/ncclimo rhea.ccs.ornl.gov:bin_rhea
# scp ~/nco/data/ncclimo skyglow.ess.uci.edu:bin
# scp ~/nco/data/ncclimo yellowstone.ucar.edu:bin
# scp dust.ess.uci.edu:nco/data/ncclimo ~/bin

# Set script name, directory, PID, run directory
drc_pwd=${PWD}
# Set these before 'module' command which can overwrite ${BASH_SOURCE[0]}
# NB: dash supports $0 syntax, not ${BASH_SOURCE[0]} syntax
# http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in
spt_src="${BASH_SOURCE[0]}"
[[ -z "${spt_src}" ]] && spt_src="${0}" # Use ${0} when BASH_SOURCE is unavailable (e.g., dash)
while [ -h "${spt_src}" ]; do # Recursively resolve ${spt_src} until file is no longer a symlink
  drc_spt="$( cd -P "$( dirname "${spt_src}" )" && pwd )"
  spt_src="$(readlink "${spt_src}")"
  [[ ${spt_src} != /* ]] && spt_src="${drc_spt}/${spt_src}" # If ${spt_src} was relative symlink, resolve it relative to path where symlink file was located
done
cmd_ln="${spt_src} ${@}"
drc_spt="$( cd -P "$( dirname "${spt_src}" )" && pwd )"
spt_nm=$(basename ${spt_src}) # [sng] Script name (Unlike $0, ${BASH_SOURCE[0]} works well with 'source <script>')
spt_pid=$$ # [nbr] Script PID (process ID)

# Configure paths at High-Performance Computer Centers (HPCCs) based on ${HOSTNAME}
if [ -z "${HOSTNAME}" ]; then
    if [ -f /bin/hostname ] && [ -x /bin/hostname ]; then
	export HOSTNAME=`/bin/hostname`
    elif [ -f /usr/bin/hostname ] && [ -x /usr/bin/hostname ]; then
	export HOSTNAME=`/usr/bin/hostname`
    fi # !hostname
fi # HOSTNAME
# Default input and output directory is ${DATA}
if [ -z "${DATA}" ]; then
    case "${HOSTNAME}" in 
	constance* | node* ) DATA='/scratch' ; ;; # PNNL
	blues* | blogin* | b590 ) DATA="/lcrc/project/ACME/${USER}" ; ;; # ALCF blues compute nodes named bNNN, 16|64 cores|GB/node 
	cooley* | cc* | mira* ) DATA="/projects/HiRes_EarthSys/${USER}" ; ;; # ALCF cooley compute nodes named ccNNN, 384 GB/node 
	cori* | edison* ) DATA="${SCRATCH}" ; ;; # NERSC cori/edison compute nodes all named nidNNNNN, edison 24|64 cores|GB/node; cori 32|128 cores|GB/node (cori login nodes 512 GB)
	pileus* ) DATA="/lustre/atlas/world-shared/cli115/${USER}" ; ;; # OLCF CADES
	rhea* | titan* ) DATA="/lustre/atlas/world-shared/cli115/${USER}" ; ;; # OLCF rhea compute nodes named rheaNNN, 128 GB/node
	ys* ) DATA="/glade/p/work/${USER}" ; ;; # NCAR yellowstone compute nodes named ysNNN, 32 GB/node
	* ) DATA='/tmp' ; ;; # Other
    esac # !HOSTNAME
fi # DATA
# Ensure batch jobs access correct 'mpirun' (or, with SLURM, 'srun') command, netCDF library, and NCO executables and library:
case "${HOSTNAME}" in 
    aims* )
	export PATH='/export/zender1/bin'\:${PATH}
        export LD_LIBRARY_PATH='/export/zender1/lib'\:${LD_LIBRARY_PATH} ; ;;
    blues* | blogin* | b590 )
	export PATH='/home/zender/bin'\:${PATH}
	export LD_LIBRARY_PATH='/home/zender/lib'\:${LD_LIBRARY_PATH} ; ;;
    cooley* )
	# 20160421: Split cooley from mira binary locations to allow for different system libraries
	# http://www.mcs.anl.gov/hs/software/systems/softenv/softenv-intro.html
	soft add +mvapich2 
        export PBS_NUM_PPN=12 # Spoof PBS on Soft (which knows nothing about node capabilities)
	export PATH='/home/zender/bin_cooley'\:${PATH}
	export LD_LIBRARY_PATH='/home/zender/lib_cooley'\:${LD_LIBRARY_PATH} ; ;;
    mira* )
	export PATH='/home/zender/bin_mira'\:${PATH}
	export LD_LIBRARY_PATH='/soft/libraries/netcdf/current/library:/home/zender/lib_mira'\:${LD_LIBRARY_PATH} ; ;;
    cori* )
	# 20160407: Split cori from edison binary locations to allow for different system libraries
	# 20160420: module load gsl, udunits required for non-interactive batch submissions by Wuyin Lin
	# Not necessary for interactive, nor for CSZ non-interactive, batch submisssions
	# Must be due to home environment differences between CSZ and other users
	# Loading gsl and udunits seems to do no harm, so always do it
	# This is equivalent to LD_LIBRARY_PATH method used for netCDF and SZIP on rhea
	# Why do cori/edison and rhea require workarounds for different packages?
	module load gsl
	module load udunits
	# On cori, module load ncl installs ERWG in ${NCARG_ROOT}/../intel/bin
	if [ -n "${NCARG_ROOT}" ]; then
            export PATH="${NCARG_ROOT}/bin:${PATH}"
	fi # !NCARG_ROOT
	export PATH='/global/homes/z/zender/bin_cori'\:${PATH}
        export LD_LIBRARY_PATH='/global/homes/z/zender/lib_cori'\:${LD_LIBRARY_PATH} ; ;;
    edison* )
	module load gsl
	module load udunits
	export PATH='/global/homes/z/zender/bin_edison'\:${PATH}
        export LD_LIBRARY_PATH='/global/homes/z/zender/lib_edison'\:${LD_LIBRARY_PATH} ; ;;
    pileus* )
	export PATH='/home/zender/bin'\:${PATH}
	export LD_LIBRARY_PATH='/opt/ACME/uvcdat-2.2-build/install/Externals/lib:/home/zender/lib'\:${LD_LIBRARY_PATH} ; ;;
    rhea* )
	# 20151017: CSZ next three lines guarantee finding mpirun
	source ${MODULESHOME}/init/sh # 20150607: PMC Ensures find module commands will be found
	module unload PE-intel # Remove Intel-compiled mpirun environment
	module load PE-gnu # Provides GCC-compiled mpirun environment (CSZ uses GCC to build NCO on rhea)
	# 20160219: CSZ UVCDAT setup causes failures with mpirun, attempting a work-around
	if [ -n "${UVCDAT_SETUP_PATH}" ]; then
	    module unload python ompi paraview PE-intel PE-gnu
	    module load gcc
	    source /lustre/atlas1/cli900/world-shared/sw/rhea/uvcdat/latest_full/bin/setup_runtime.sh
	    export ${UVCDAT_SETUP_PATH}
	fi # !UVCDAT_SETUP_PATH
	# On rhea, module load ncl installs ERWG in ${NCL_DIR}/bin
	if [ -n "${NCL_DIR}" ]; then
            export PATH="${NCL_DIR}/bin:${PATH}"
	fi # !NCL_DIR
        export PATH='/ccs/home/zender/bin_rhea'\:${PATH}
	export LD_LIBRARY_PATH='/sw/redhat6/netcdf/4.3.3.1/rhel6.6_gcc4.8.2--with-dap+hdf4/lib:/sw/redhat6/szip/2.1/rhel6.6_gnu4.8.2/lib:/ccs/home/zender/lib_rhea'\:${LD_LIBRARY_PATH} ; ;;
    titan* )
	source ${MODULESHOME}/init/sh # 20150607: PMC Ensures find module commands will be found
	module load gcc
        export PATH='/ccs/home/zender/bin_titan'\:${PATH}
	export LD_LIBRARY_PATH='/opt/cray/netcdf/4.3.2/GNU/49/lib:/sw/xk6/udunits/2.1.24/sl_gcc4.5.3/lib:/ccs/home/zender/lib_titan'\:${LD_LIBRARY_PATH} ; ;;
    ys* )
	# 20151018: Yellowstone support not yet tested in batch mode
	# On yellowstone, module load ncl installs ERWG in /glade/apps/opt/ncl/6.3.0/intel/12.1.5/bin (not in ${NCARG_ROOT}/bin)
	if [ -n "${NCARG_ROOT}" ]; then
#            export PATH="${NCARG_ROOT}/bin:${PATH}"
            export PATH="${PATH}:/glade/apps/opt/ncl/6.3.0/intel/12.1.5/bin"
	fi # !NCARG_ROOT
        export PATH='/glade/u/home/zender/bin'\:${PATH}
        export LD_LIBRARY_PATH='/glade/apps/opt/netcdf/4.3.0/intel/12.1.5/lib:/glade/u/home/zender/lib'\:${LD_LIBRARY_PATH}
esac # !HOSTNAME

# Production usage:
# ncclimo -c famipc5_ne30_v0.3_00003 -s 1980 -e 1983 -i /lustre/atlas1/cli115/world-shared/mbranst/famipc5_ne30_v0.3_00003-wget-test -o ${DATA}/ne30/clm
# ncclimo -c famipc5_ne120_v0.3_00003 -s 1980 -e 1983 -i /lustre/atlas1/cli115/world-shared/mbranst/famipc5_ne120_v0.3_00003-wget-test -o ${DATA}/ne120/clm
# ncclimo -c B1850C5e1_ne30 -s 2 -e 199 -i /lustre/atlas1/cli115/world-shared/mbranst/B1850C5e1_ne30/atm/hist -o ${DATA}/ne30/clm

# Incremental climo testing:
# ncclimo -v FSNT,AODVIS -c famipc5_ne30_v0.3_00003 -s 1980 -e 1981 -i ${DATA}/ne30/raw -o ${DATA}/ne30/prv -r ${DATA}/maps/map_ne30np4_to_fv129x256_aave.20150901.nc
# ncclimo -v FSNT,AODVIS -c famipc5_ne30_v0.3_00003 -s 1982 -e 1983 -i ${DATA}/ne30/raw -o ${DATA}/ne30/clm -r ${DATA}/maps/map_ne30np4_to_fv129x256_aave.20150901.nc -x ${DATA}/ne30/prv -X ${DATA}/ne30/xtn -S 1980
# Binary climo testing:
# ncclimo -v FSNT,AODVIS -c famipc5_ne30_v0.3_00003 -S 1980 -E 1981 -x ${DATA}/ne30/prv -s 1982 -e 1983 -i ${DATA}/ne30/clm -X ${DATA}/ne30/xtn

# Annual climo testing:
# ncclimo -C ann -m cism -h h -c b.e10.BG20TRCN.f09_g16.002 -s 1851 -e 2006 -i /lustre/atlas1/cli115/proj-shared/4ue/data/for_charlie -o ${DATA}/ne30/clm
# ncclimo -C ann -m cism -h h -c b.e10.BG20TRCN.f09_g16.002 -s 1851 -e 1852 -i /lustre/atlas1/cli115/proj-shared/4ue/data/for_charlie -o ${DATA}/ne30/clm > ~/ncclimo.out 2>&1 &

# Debugging and Benchmarking:
# ncclimo -v FSNT,AODVIS -c famipc5_ne30_v0.3_00003 -s 1980 -e 1983 -i ${DATA}/ne30/raw -o ${DATA}/ne30/clm -r ${DATA}/maps/map_ne30np4_to_fv129x256_aave.20150901.nc
# ncclimo --var_lst=FSNT,AODVIS --caseid=famipc5_ne30_v0.3_00003 --yr_srt=1980 --yr_end=1983 --drc_in=${DATA}/ne30/raw --drc_out=${DATA}/ne30/clm --rgr_map=${DATA}/maps/map_ne30np4_to_fv129x256_aave.20150901.nc
# ncclimo -v TOTEXTTAU -c merra2_198001.nc4 -s 1980 -e 2015 -a sdd -i ${DATA}/merra2/raw -o ${DATA}/merra2/clm
# ncclimo > ~/ncclimo.out 2>&1 &
# ncclimo -c B1850C5e1_ne30 -s 2 -e 199 > ~/ncclimo.out 2>&1 &
# ncclimo -c ne30_gx1.B1850c5d -s 6 -e 7 > ~/ncclimo.out 2>&1 &
# ncclimo -d 2 -v FSNT -m cam2 -c essgcm14 -s 1 -e 20 -i ${DATA}/essgcm14 -o ${DATA}/anl > ~/ncclimo.out 2>&1 &
# ncclimo -c famipc5_ne30_v0.3_00003 -s 1980 -e 1983 -i /lustre/atlas1/cli115/world-shared/mbranst/famipc5_ne30_v0.3_00003-wget-test -o ${DATA}/ne30/clm > ~/ncclimo.out 2>&1 &
# ncclimo -c famipc5_ne120_v0.3_00003 -s 1980 -e 1983 -i /lustre/atlas1/cli115/world-shared/mbranst/famipc5_ne120_v0.3_00003-wget-test -o ${DATA}/ne120/clm > ~/ncclimo.out 2>&1 &
# MPAS: Prior to running ncclimo on MPAS output, annotate missing values of input with, e.g.,
# for fl in `ls hist.*` ; do
#  ncatted -O -t -a _FillValue,,o,d,-9.99999979021476795361e+33 ${fl}
# done
# New MPAS filename conventions (as of ~201612):
# ncclimo -v timeMonthly_avg_activeTracers_temperature -s 2 -e 3 -m mpaso    -i /scratch2/scratchdirs/golaz/ACME_simulations/20161117.beta0.A_WCYCL1850S.ne30_oEC_ICG.edison/run -r ${DATA}/maps/map_oEC60to30_to_t62_bilin.20160301.nc -o ${DATA}/mpas/clm > ~/ncclimo.out 2>&1 &
 # ncclimo -v timeSeriesStatsMonthly_avg_iceAreaCell_1 -s 2 -e 3 -m mpascice -i /scratch2/scratchdirs/golaz/ACME_simulations/20161117.beta0.A_WCYCL1850S.ne30_oEC_ICG.edison/run -r ${DATA}/maps/map_oEC60to30_to_t62_bilin.20160301.nc -o ${DATA}/mpas/clm > ~/ncclimo.out 2>&1 &
# Old MPAS filename conventions (until ~201609)::
# ncclimo -v temperature -c hist -s 2 -e 3 -m ocn -i /lustre/atlas1/cli112/proj-shared/golaz/ACME_simulations/20160121.A_B2000ATMMOD.ne30_oEC.titan.a00/run -r ${DATA}/maps/map_oEC60to30_to_t62_bilin.20160301.nc -o ${DATA}/mpas/clm > ~/ncclimo.out 2>&1 &
# ncclimo -v iceAreaCell -c hist -s 2 -e 3 -m ice -i /lustre/atlas1/cli112/proj-shared/golaz/ACME_simulations/20160121.A_B2000ATMMOD.ne30_oEC.titan.a00/run -r ${DATA}/maps/map_oEC60to30_to_t62_bilin.20160301.nc -o ${DATA}/mpas/clm > ~/ncclimo.out 2>&1 &

# Best performance on resolutions finer than ne30 (~1x1 degree) requires a job scheduler/batch processor
# Cobalt (cooley), SLURM (cori,edison), Maui (a PBS-variant) (blues), Torque (a PBS-variant) (hopper), and PBS (rhea) schedulers allow both interactive and non-interactive (i.e., script) batch jobs
# ALCF Maui:
# http://www.lcrc.anl.gov/for-users/using-lcrc/running-jobs
# ALCF Cobalt:
# softenv # lists available packages
# http://www.mcs.anl.gov/hs/software/systems/softenv/softenv-intro.html
# http://www.alcf.anl.gov/user-guides/using-cobalt-cooley
# https://www.alcf.anl.gov/user-guides/cobalt-job-control
# NERSC SLURM:
# https://www.nersc.gov/users/computational-systems/cori/running-jobs/slurm-introduction
# https://www.nersc.gov/users/computational-systems/cori/running-jobs/for-edison-users/torque-moab-vs-slurm-comparisons
# https://www.nersc.gov/users/computational-systems/cori/running-jobs/queues-and-policies/
# https://www.nersc.gov/users/computational-systems/edison/running-jobs/queues-and-policies/
# OLCF PBS: 
# https://www.olcf.ornl.gov/support/system-user-guides/rhea-user-guide/#903
# Requesting interactive nodes, Submitting non-interactive batch jobs, Monitoring queues, Deleting jobs:
# Cobalt: qsub -I,   qsub,  qstat,    qdel
# PBS:    qsub -I,   qsub,  qstat,    qdel
# SLURM:   salloc, sbatch, squeue, scancel
# Interactive queue: a) Reserve nodes and acquire prompt on control node b) Execute ncclimo command interactively
#   Blues:  qsub -I -A ACME -q acme -l nodes=12 -l walltime=00:30:00 -N ncclimo
#   Cooley: qsub -I -A HiRes_EarthSys --nodecount=12 --time=00:30:00 --jobname=ncclimo
#   Cori:   salloc  -A acme --nodes=12 --partition=debug --time=00:30:00 --job-name=ncclimo # NB: 30 minute limit, Edison too
#   Hopper: qsub -I -A acme -V -l mppwidth=288 -l walltime=00:30:00 -q debug -N ncclimo # deprecated, old Edison
#   Rhea:   qsub -I -A CLI115 -V -l nodes=12 -l walltime=00:30:00 -N ncclimo # Bigmem: -l partition=gpu
#   Yellow: fxm # Bigmem: 
# Non-interactive batch procedure: a) Store ncclimo command in ncclimo.[cobalt|pbs|slurm] b) qsub ncclimo.[cobalt|pbs|slurm]
# Non-interactive batch queue differences (besides argument syntax):
# 1. Cobalt and SLURM require initial 'shebang' line to specify the shell interpreter (not required on PBS)
# 2. Cobalt appends stdout/stderr to existing output files, if any, whereas PBS overwrites existing files
# 3. Cobalt uses ${COBALT_NODEFILE} and (NA) whereas PBS uses ${PBS_NODEFILE} and ${PBS_NUM_PPN}, respectively, and SLURM uses ${SLURM_NODELIST} and ${SLURM_CPUS_ON_NODE}, respectively
# 4. SLURM automatically combines stdout and stderr, yet does not understand tilde (~ = home directory) expansion in error/output filenames
# Differences 1 & 2 impose slightly different invocations; difference 3 requires abstracting environment variables; difference 4 requires omitting ~'s
#   Cooley a): /bin/rm -f ~/ncclimo.err  ~/ncclimo.out
#              echo '#!/bin/bash' > ~/ncclimo.cobalt
#              echo "ncclimo -d 1 -p mpi -c b1850c5_m2a -s 55 -e 58 -i /home/taylorm/scratch1.qtang/b1850c5_m2a/run -o ${DATA}/ne120/clm" >> ~/ncclimo.cobalt;chmod a+x ~/ncclimo.cobalt
#   Cori,Edison a): echo '#!/bin/bash' > ~/ncclimo.slurm
#                   echo "ncclimo -a scd -d 1 -p mpi -c famipc5_ne30_v0.3_00003 -s 1980 -e 1983 -i ${DATA}/ne30/raw -o ${DATA}/ne30/clm -r ${DATA}/maps/map_ne30np4_to_fv129x256_aave.20150901.nc" >> ~/ncclimo.slurm;chmod a+x ~/ncclimo.slurm
#   Rhea a):   echo "ncclimo -a scd -d 1 -p mpi -c famipc5_ne120_v0.3_00003 -s 1980 -e 1983 -i /lustre/atlas1/cli115/world-shared/mbranst/famipc5_ne120_v0.3_00003-wget-test -o ${DATA}/ne120/clm -r ${DATA}/maps/map_ne120np4_to_fv257x512_aave.20150901.nc"  > ~/ncclimo.pbs;chmod a+x ~/ncclimo.pbs
#   Cooley b): qsub -A HiRes_EarthSys --nodecount=12 --time=00:30:00 --jobname ncclimo --error ~/ncclimo.err --output ~/ncclimo.out --notify zender@uci.edu ~/ncclimo.cobalt
#   Cori,Edison b): sbatch -A acme --nodes=12 --time=00:30:00 --partition=regular --job-name=ncclimo --mail-type=END --output=ncclimo.out ~/ncclimo.slurm
#   Hopper b): qsub -A acme -V -l mppwidth=288 -l walltime=00:30:00 -q regular -N ncclimo -j oe -m e -o ~/ncclimo.out ~/ncclimo.pbs
#   Rhea b):   qsub -A CLI115 -V -l nodes=12 -l walltime=00:30:00 -N ncclimo -j oe -m e -o ~/ncclimo.out ~/ncclimo.pbs

# Normal use: Set five "mandatory" inputs (caseid, yr_srt, yr_end, drc_in, drc_out), and possibly rgr_map, on command line
# caseid:  Simulation name (filenames must start with ${caseid})
# drc_in:  Input directory for raw data
#          Years outside yr_srt and yr_end are ignored
#          yr_srt should, and for SDD mode must, contain complete year of output
#          SCD mode ignores Jan-Nov of yr_srt
#          Dec of yr_end is excluded from the seasonal and monthly analysis in SCD mode
#          yr_end should, and for SDD mode must, contain complete year of output
# drc_out: Output directory for processed native grid climatology ("climo files")
#          User needs write permission for ${drc_out}
# rgr_map: Regridding map, if non-NULL, invoke regridder with specified map on output datasets
#          Pass options intended exclusively for the NCO regridder as arguments to the -R switch
# yr_srt:  Year of first January to analyze
# yr_end:  Year of last  January to analyze

# Other options (often their default settings work well):
# dec_md:  December mode, i.e., how to treat December. One of two options:
#          Seasonally-contiguous-december (SCD) mode (dec_md=scd) (default)
#          Seasonally-discontiguous-december (SDD) mode (dec_md=sdd)
#          Both modes use an integral multiple of 12 months, and _never alter any input files_
#          SCD climatologies begin in Dec of yr_srt-1, and end in Nov of yr_end
#          SDD climatologies begin in Jan of yr_srt,   and end in Dec of yr_end
#          SCD excludes Jan-Nov of yr_srt-1 and Dec of yr_end (i.e., SCD excludes 12 months of available data)
#          SDD uses all months of yr_srt through yr_end (i.e., SDD can use all available data)
#          SCD seasonal averages are inconsistent with (calendar-year-based) annual averages, but better capture seasonal the "natural" (not calendar-year-based) climate year
#          SDD seasonal averages are fully consistent with (calendar-year-based) annual averages
# drc_rgr: Regridding directory---store regridded files, if any, in drc_rgr rather than drc_out
# lnk_flg: Link ACME-climo to AMWG-climo filenames
#          AMWG omits the YYYYMM components of climo filenames, resulting in shorter names
#          This switch (on by default) symbolically links the full (ACME) filename to the shorter (AMWG) name
#          AMWG diagnostics scripts can produce plots directly from these linked filenames
# par_typ: Parallelism type---all values _except_ exact matches to "bck" and "mpi" are interpreted as "nil" (and invoke serial mode)
#          bck = Background: Spawn children (basic blocks) as background processes on control node then wait()
#                Works best when available RAM > 12*4*sizeof(monthly input file), otherwise jobs swap-to-disk
#          mpi = MPI: Spawn children (basic blocks) as MPI processes (one per node in batch environment) then wait()
#                Requires batch system with PBS and MPI. Use when available RAM/node < 12*2.5*sizeof(monthly input file).
#                Optimized for batch with 12 nodes. Factors thereof (6, 4, 3, 2 nodes) should also work.
#                Remember to request 12 nodes if possible!
#          nil = None: Execute script in serial mode on single node
#                Works best when available RAM < 12*4*sizeof(monthly input file), otherwise jobs swap-to-disk
# var_lst: Variables to include, or, with nco_opt='-x', to exclude, in comma-separated list format, e.g.,
#          'FSNT,AODVIS'. Regular expressions work, too: 'AOD.?'

# Infrequently used options:
# dbg_lvl: 0 = Quiet, print basic status during evaluation
#          1 = Print configuration, full commands, and status to output during evaluation
#          2 = As in dbg_lvl=1, but do not evaluate commands
#          3 = As in dbg_lvl=2, with additional information (mainly for batch queues)
# fml_nm:  Family name (nickname) of output files referring to $fml_nm character sequence used in output climo file names:
#          fml_nm_XX_YYYYMM_YYYYMM.nc (examples include '' (default), 'control', 'experiment')
#          By default, fml_nm=$caseid. Use fml_nm instead of $caseid to simplify long names, avoid overlap, etc.
# hst_nm:  History volume name referring to the $hst_nm character sequence used in history tape names:
#          caseid.mdl_nm.hst_nm.YYYY-MM.nc (examples include 'h0' (default, works for cam, clm), 'h1', 'h' (for cism), 'hist' (for mpascice, mpaso)
# mdl_nm:  Model name referring to the character sequence $mdl_nm used in history tape names:
#          caseid.mdl_nm.h0.YYYY-MM.nc (examples include 'cam' (default), 'clm2', 'cam2', 'cism', 'mpaso', 'mpascice', 'pop')
# nco_opt: String of options to pass-through to NCO, e.g.,
#          '-D 2 -7 -L 1' for NCO debugging level 2, netCDF4-classic output, compression level 1
#          '--no_tmp_fl -x' to skip temporary files, turn extraction into exclusion list
# rgr_opt: String of options (besides thread-number) to pass-through exclusively to NCO regridder, e.g., 
#          ncclimo -m clm2 ... -R col_nm=lndgrid -r map.nc ...
# thr_nbr: Thread number to use in NCO regridder, '-t 1' for one thread, '-t 2' for two threads...

# Set NCO version and directory
nco_exe=`which ncks`
if [ -z "${nco_exe}" ]; then
    echo "ERROR: Unable to find NCO, nco_exe = ${nco_exe}"
    exit 1
fi # !nco_exe
# Use StackOverflow method to find NCO directory
while [ -h "${nco_exe}" ]; do
  drc_nco="$( cd -P "$( dirname "${nco_exe}" )" && pwd )"
  nco_exe="$(readlink "${nco_exe}")"
  [[ ${nco_exe} != /* ]] && nco_exe="${drc_nco}/${nco_exe}"
done
drc_nco="$( cd -P "$( dirname "${nco_exe}" )" && pwd )"
nco_vrs=$(ncks --version 2>&1 >/dev/null | grep NCO | awk '{print $5}')

# When running in a terminal window (not in an non-interactive batch queue)...
if [ -n "${TERM}" ]; then
    # Set fonts for legibility
    if [ -x /usr/bin/tput ] && tput setaf 1 &> /dev/null; then
	fnt_bld=`tput bold` # Bold
	fnt_nrm=`tput sgr0` # Normal
	fnt_rvr=`tput smso` # Reverse
	fnt_tlc=`tput sitm` # Italic
    else
	fnt_bld="\e[1m" # Bold
	fnt_nrm="\e[0m" # Normal
	fnt_rvr="\e[07m" # Reverse
	fnt_tlc="\e[3m" # Italic
    fi # !tput
fi # !TERM
    
# Defaults for command-line options and some derived variables
# Modify these defaults to save typing later
ann_sfx='01-01-00000' # [sng] Annual file suffix (e.g., '01-01-00000')
bnr_flg='No' # [sng] Binary method
caseid='' # [sng] Case ID
caseid_xmp='famipc5_ne30_v0.3_00003' # [sng] Case ID for examples
cf_flg='Yes' # [sng] Produce CF climatology attribute?
clm_md='mth' # [sng] Climatology mode ('ann', 'dly', or 'mth')
dbg_lvl=0 # [nbr] Debugging level
dec_md='scd' # [sng] December mode ('scd' or 'sdd' as per above)
drc_in='' # [sng] Input file directory
drc_in_xmp="${DATA}/ne30/raw" # [sng] Input file directory for examples
drc_in_mps="${DATA}/mpas/raw" # [sng] Input file directory for MPAS examples
drc_out='' # [sng] Output file directory
drc_out_xmp="${DATA}/ne30/clm" # [sng] Output file directory for examples
drc_out_mps="${DATA}/mpas/clm" # [sng] Output file directory for MPAS examples
drc_prv='' # [sng] Directory containing previous climatology to extend with current data
drc_rgr='' # [sng] Regridded file directory
drc_rgr_prv='' # [sng] Regridded file directory for previous climatology
drc_rgr_xmp="${DATA}/ne30/rgr" # [sng] Regrid file directory for examples
drc_rgr_xtn='' # [sng] Regridded file directory for for extended climatology
drc_xtn='' # [sng] Directory containing extended climatology
fml_nm='' # [sng] Family name (i.e., nickname, e.g., 'amip', 'control', 'experiment')
gaa_sng_std="--gaa climo_script=${spt_nm} --gaa climo_command=\"'${cmd_ln}'\" --gaa climo_hostname=${HOSTNAME} --gaa climo_version=${nco_vrs}" # [sng] Global attributes to add
hdr_pad='10000' # [B] Pad at end of header section
hst_nm='h0' # [sng] History volume (e.g., 'h0', 'h1', 'h')
lnk_flg='Yes' # [sng] Link ACME-climo to AMWG-climo filenames
mdl_nm='cam' # [sng] Model name (e.g., 'cam', 'cam2', 'cice', 'cism', 'clm', 'clm2', 'ice', 'mpascice', 'mpaso', 'ocn')
mdl_typ='cesm' # [sng] Model type ('cesm', 'mpas') (for filenames and regridding)
mpi_flg='No' # [sng] Parallelize over nodes
nco_opt='--no_tmp_fl' # [sng] NCO options (e.g., '-7 -D 1 -L 1')
ncr_flg='No' # [sng] Incremental method
nd_nbr=1 # [nbr] Number of nodes
par_opt='' # [sng] Parallel options to shell
par_typ='bck' # [sng] Parallelism type
rgr_map='' # [sng] Regridding map
#rgr_map="${DATA}/maps/map_ne30np4_to_fv129x256_aave.20150901.nc"
#rgr_map="${DATA}/maps/map_ne30np4_to_fv257x512_bilin.20150901.nc"
#rgr_map="${DATA}/maps/map_ne120np4_to_fv257x512_aave.20150901.nc"
#rgr_map="${DATA}/maps/map_ne120np4_to_fv801x1600_bilin.20150901.nc"
rgr_opt='' # [sng] Regridding options (e.g., '--rgr col_nm=lndgrid', '--rgr col_nm=nCells')
thr_nbr=2 # [nbr] Thread number for regridder
#var_lst='FSNT,AODVIS' # [sng] Variables to process (empty means all)
var_lst='' # [sng] Variables to process (empty means all)
xtn_flg='No' # [sng] Produce extended climatology
yr_end='1983' # [yr] End year
yr_srt='1980' # [yr] Start year

function fnc_usg_prn { # NB: dash supports fnc_nm (){} syntax, not function fnc_nm{} syntax
    # Print usage
    printf "${fnt_rvr}Basic usage:\n${fnt_nrm} ${fnt_bld}${spt_nm} -c caseid -s yr_srt -e yr_end -i drc_in -o drc_out -r rgr_map${fnt_nrm}\n"
    printf "${fnt_nrm} ${fnt_bld}${spt_nm} --case=caseid --start_year=yr_srt --end_year=yr_end --input=drc_in --output=drc_out --map=rgr_map${fnt_nrm}\n\n"
    echo "Command-line options [long-option synonyms in ${fnt_tlc}italics${fnt_nrm}]:"
    echo "${fnt_rvr}-a${fnt_nrm} ${fnt_bld}dec_md${fnt_nrm}   December mode (default ${fnt_bld}${dec_md}${fnt_nrm}) [${fnt_tlc}dec_md, december_mode, dec_mode${fnt_nrm}]"
    echo "${fnt_rvr}-C${fnt_nrm} ${fnt_bld}clm_md${fnt_nrm}   Climatology mode (default ${fnt_bld}${clm_md}${fnt_nrm}) [${fnt_tlc}clm_md, climatology_mode, climo_mode${fnt_nrm}]"
    echo "${fnt_rvr}-c${fnt_nrm} ${fnt_bld}caseid${fnt_nrm}   Case ID string (default ${fnt_bld}${caseid}${fnt_nrm}) [${fnt_tlc}caseid, case_id, case${fnt_nrm}]"
    echo "${fnt_rvr}-d${fnt_nrm} ${fnt_bld}dbg_lvl${fnt_nrm}  Debug level (default ${fnt_bld}${dbg_lvl}${fnt_nrm}) [${fnt_tlc}dbg_lvl, dbg, debug, debug_level${fnt_nrm}]"
    echo "${fnt_rvr}-E${fnt_nrm} ${fnt_bld}yr_end${fnt_nrm}   End year previous climo (empty means none) (default ${fnt_bld}${yr_end_prv}${fnt_nrm}) [${fnt_tlc}yr_end_prv, prv_yr_end, previous_end${fnt_nrm}]"
    echo "${fnt_rvr}-e${fnt_nrm} ${fnt_bld}yr_end${fnt_nrm}   End year (default ${fnt_bld}${yr_end}${fnt_nrm}) [${fnt_tlc}yr_end, end_yr, year_end, end_year, end${fnt_nrm}]"
    echo "${fnt_rvr}-f${fnt_nrm} ${fnt_bld}fml_nm${fnt_nrm}   Family name (nickname) (empty means none) (default ${fnt_bld}${fml_nm}${fnt_nrm}) [${fnt_tlc}fml_nm, family_name${fnt_nrm}]"
    echo "${fnt_rvr}-h${fnt_nrm} ${fnt_bld}hst_nm${fnt_nrm}   History volume name (default ${fnt_bld}${hst_nm}${fnt_nrm}) [${fnt_tlc}hst_nm, history_name, history${fnt_nrm}]"
    echo "${fnt_rvr}-i${fnt_nrm} ${fnt_bld}drc_in${fnt_nrm}   Input directory (default ${fnt_bld}${drc_in}${fnt_nrm}) [${fnt_tlc}drc_in, in_drc, dir_in, in_dir, input${fnt_nrm}]"
    echo "${fnt_rvr}-l${fnt_nrm} ${fnt_bld}lnk_flg${fnt_nrm}  Link ACME-climo to AMWG-climo filenames (default ${fnt_bld}${lnk_flg}${fnt_nrm}) [${fnt_tlc}lnk_flg, link_flag, no_amwg_links${fnt_nrm}]"
    echo "${fnt_rvr}-m${fnt_nrm} ${fnt_bld}mdl_nm${fnt_nrm}   Model name (default ${fnt_bld}${mdl_nm}${fnt_nrm}) [${fnt_tlc}mdl_nm, model_name, model${fnt_nrm}]"
    echo "${fnt_rvr}-n${fnt_nrm} ${fnt_bld}nco_opt${fnt_nrm}  NCO options (empty means none) (default ${fnt_bld}${nco_opt}${fnt_nrm}) [${fnt_tlc}nco_opt, nco, nco_options${fnt_nrm}]"
    echo "${fnt_rvr}-O${fnt_nrm} ${fnt_bld}drc_rgr${fnt_nrm}  Regridded directory (default ${fnt_bld}${drc_rgr}${fnt_nrm}) [${fnt_tlc}drc_rgr, rgr_drc, dir_regrid, regrid${fnt_nrm}]"
    echo "${fnt_rvr}-o${fnt_nrm} ${fnt_bld}drc_out${fnt_nrm}  Output directory (default ${fnt_bld}${drc_out}${fnt_nrm}) [${fnt_tlc}drc_out, out_drc, dir_out, out_dir, output${fnt_nrm}]"
    echo "${fnt_rvr}-p${fnt_nrm} ${fnt_bld}par_typ${fnt_nrm}  Parallelism type (default ${fnt_bld}${par_typ}${fnt_nrm}) [${fnt_tlc}par_typ, par_md, parallel_type, parallel_mode, parallel${fnt_nrm}]"
    echo "${fnt_rvr}-r${fnt_nrm} ${fnt_bld}rgr_map${fnt_nrm}  Regrid map (empty means none) (default ${fnt_bld}${rgr_map}${fnt_nrm}) [${fnt_tlc}rgr_map, regrid_map$, map, map_file, map_fl{fnt_nrm}]"
    echo "${fnt_rvr}-R${fnt_nrm} ${fnt_bld}rgr_opt${fnt_nrm}  Regrid options (empty means none) (default ${fnt_bld}${rgr_opt}${fnt_nrm}) [${fnt_tlc}rgr_opt, regrid_options${fnt_nrm}]"
    echo "${fnt_rvr}-t${fnt_nrm} ${fnt_bld}thr_nbr${fnt_nrm}  Thread number for regridder (default ${fnt_bld}${thr_nbr}${fnt_nrm}) [${fnt_tlc}thr_nbr, thread_number, thread, threads${fnt_nrm}]"
    echo "${fnt_rvr}-S${fnt_nrm} ${fnt_bld}yr_srt${fnt_nrm}   Start year previous climo (empty means none) (default ${fnt_bld}${yr_srt_prv}${fnt_nrm}) [${fnt_tlc}yr_srt_prv, prv_yr_srt, previous_start${fnt_nrm}]"
    echo "${fnt_rvr}-s${fnt_nrm} ${fnt_bld}yr_srt${fnt_nrm}   Start year (default ${fnt_bld}${yr_srt}${fnt_nrm}) [${fnt_tlc}yr_srt, start_yr, year_start, start_year, start${fnt_nrm}]"
    echo "${fnt_rvr}-v${fnt_nrm} ${fnt_bld}var_lst${fnt_nrm}  Variable list (empty means all) (default ${fnt_bld}${var_lst}${fnt_nrm}) [${fnt_tlc}var_lst, variable_list, variable, variables${fnt_nrm}]"
    echo "${fnt_rvr}-X${fnt_nrm} ${fnt_bld}drc_xtn${fnt_nrm}  Extended climo directory (default ${fnt_bld}${drc_xtn}${fnt_nrm}) [${fnt_tlc}drc_xtn, xtn_drc, extended_dir, extended_climo, extended${fnt_nrm}]"
    echo "${fnt_rvr}-x${fnt_nrm} ${fnt_bld}drc_prv${fnt_nrm}  Previous climo directory (default ${fnt_bld}${drc_prv}${fnt_nrm}) [${fnt_tlc}drc_prv, prv_drc, previous_dir, previous_climo, previous${fnt_nrm}]"
    echo "${fnt_rvr}-Y${fnt_nrm} ${fnt_bld}rgr_xtn${fnt_nrm}  Regridded extended climo directory (default ${fnt_bld}${drc_rgr_xtn}${fnt_nrm}) [${fnt_tlc}drc_rgr_xtn, drc_xtn_rgr, regridded_extended, extended_regridded${fnt_nrm}]"
    echo "${fnt_rvr}-y${fnt_nrm} ${fnt_bld}rgr_prv${fnt_nrm}  Regridded previous climo directory (default ${fnt_bld}${drc_rgr_prv}${fnt_nrm}) [${fnt_tlc}drc_rgr_prv, drc_prv_rgr, regridded_previous, prefvious_regridded${fnt_nrm}]"
    printf "\n"
    printf "${fnt_rvr}Examples:${fnt_nrm}\n${fnt_bld}${spt_nm} -c ${caseid_xmp} -s ${yr_srt} -e ${yr_end} -i ${drc_in_xmp} -o ${drc_out_xmp} ${fnt_nrm}\n"
    printf "${fnt_bld}${spt_nm} -c ${caseid_xmp} -s ${yr_srt} -e ${yr_end} -i ${drc_in_xmp} -o ${drc_out_xmp} -r ~zender/data/maps/map_ne30np4_to_fv129x256_aave.20150901.nc ${fnt_nrm}\n"
    printf "${fnt_bld}${spt_nm} -c control -m clm2 -s ${yr_srt} -e ${yr_end} -i ${drc_in_xmp} -o ${drc_out_xmp} -r ~zender/data/maps/map_ne30np4_to_fv129x256_aave.20150901.nc ${fnt_nrm}\n"
    printf "${fnt_bld}${spt_nm} -m mpascice  -s ${yr_srt} -e ${yr_end} -i ${drc_in_mps} -o ${drc_out_mps} -r ~zender/data/maps/map_oEC60to30_to_t62_bilin.20160301.nc ${fnt_nrm}\n"
    printf "${fnt_bld}${spt_nm} -m mpaso -p mpi -s 1 -e 5 -i ${drc_in_mps} -o ${drc_out_mps} -r ~zender/data/maps/map_oEC60to30_to_t62_bilin.20160301.nc ${fnt_nrm}\n\n"
    printf "${fnt_rvr}Interactive batch queues:${fnt_nrm}\n"
    printf "blues : qsub -I -A ACME -q acme -l nodes=1 -l walltime=00:30:00 -N ncclimo\n"
    printf "cooley: qsub -I -A HiRes_EarthSys --nodecount=1 --time=00:30:00 --jobname=ncclimo\n"
    printf "cori  : salloc  -A acme --nodes=1 --time=00:30:00 --partition=debug --job-name=ncclimo\n"
    printf "edison: salloc  -A acme --nodes=1 --time=00:30:00 --partition=debug --job-name=ncclimo\n"
    printf "rhea  : qsub -I -A CLI115 -V -l nodes=1 -l walltime=00:30:00 -N ncclimo\n"
    printf "rhea  : qsub -I -A CLI115 -V -l nodes=1 -l walltime=00:30:00 -lpartition=gpu -N ncclimo # Bigmem\n"
    printf "\nComplete documentation at http://nco.sf.net/nco.html#${spt_nm}\n\n"
#    echo "3-yrs  ne30: ncclimo -c famipc5_ne30_v0.3_00003 -s 1980 -e 1982 -i /lustre/atlas1/cli115/world-shared/mbranst/famipc5_ne30_v0.3_00003-wget-test -o ${DATA}/ne30/clm -r ~zender/data/maps/map_ne30np4_to_fv129x256_aave.20150901.nc > ~/ncclimo.out 2>&1 &"
#    printf "3-yrs ne120: ncclimo -p mpi -c famipc5_ne120_v0.3_00003 -s 1980 -e 1982 -i /lustre/atlas1/cli115/world-shared/mbranst/famipc5_ne120_v0.3_00003-wget-test -o ${DATA}/ne120/clm -r ~zender/data/maps/map_ne120np4_to_fv257x512_aave.20150901.nc > ~/ncclimo.out 2>&1 &\n\n"
    exit 1
} # end fnc_usg_prn()

function trim_leading_zeros {
    # Purpose: Trim leading zeros from string representing an integer
    # Why, you ask? Because Bash treats zero-padded integers as octal!
    # This is surprisingly hard to workaround
    # My workaround is to remove leading zeros prior to arithmetic
    # Usage: trim_leading zeros ${sng}
    sng_trm=${1} # [sng] Trimmed string
    # Use Bash 2.X pattern matching to remove up to three leading zeros, one at a time
    sng_trm=${sng_trm##0} # NeR98 p. 99
    sng_trm=${sng_trm##0}
    sng_trm=${sng_trm##0}
    # If all zeros removed, replace with single zero
    if [ ${sng_trm} = '' ]; then 
	sng_trm='0'
    fi # endif
} # end trim_leading_zeros()

get_spt_drc () {
# SMB (20150814):
# Get calling script location to call other utilities in the PreAndPostProcessingScripts package
# Resolve symlinks in case script is linked elsewhere with technique from
# http://www.ostricher.com/2014/10/the-right-way-to-get-the-directory-of-a-bash-script
    spt_src="${BASH_SOURCE[0]}"
    # If ${spt_src} is a symlink, resolve it
    while [ -h "${spt_src}" ]; do
	spt_drc="$(cd -P "$(dirname "${spt_src}")" && pwd)"
        spt_src="$(readlink "${spt_src}")"
        # Resolve relative symlinks (no initial "/") against symlink base directory
        [[ ${spt_src} != /* ]] && spt_src="${spt_drc}/${spt_src}"
    done
    spt_drc="$(cd -P "$(dirname "${spt_src}")" && pwd)"
    echo ${spt_drc}
} # end get_spt_drc()

# Check argument number and complain accordingly
arg_nbr=$#
#printf "\ndbg: Number of arguments: ${arg_nbr}"
if [ ${arg_nbr} -eq 0 ]; then
  fnc_usg_prn
fi # !arg_nbr

# Parse command-line options:
# http://stackoverflow.com/questions/402377/using-getopts-in-bash-shell-script-to-get-long-and-short-command-line-options (see method by Adam Katz)
# http://tuxtweaks.com/2014/05/bash-getopts
while getopts :a:C:c:d:E:e:f:h:i:l:m:n:O:o:p:R:r:S:s:t:v:X:x:Y:y:-: OPT; do
    case ${OPT} in
	a) dec_md="${OPTARG}" ;; # December mode
	C) clm_md="${OPTARG}" ;; # Climatology mode
	c) caseid="${OPTARG}" ;; # CASEID
	d) dbg_lvl="${OPTARG}" ;; # Debugging level
	E) yr_end_prv="${OPTARG}" ;; # End year previous
	e) yr_end="${OPTARG}" ;; # End year
	f) fml_nm="${OPTARG}" ;; # Family name
	h) hst_nm="${OPTARG}" ;; # History tape name
	i) drc_in="${OPTARG}" ;; # Input directory
	l) lnk_flg="${OPTARG}" ;; # Link ACME to AMWG name
	m) mdl_nm="${OPTARG}" ;; # Model name
	n) nco_opt="${OPTARG}" ;; # NCO options
	o) drc_out_usr="${OPTARG}" ;; # Output directory
	O) drc_rgr_usr="${OPTARG}" ;; # Regridded directory
	p) par_typ="${OPTARG}" ;; # Parallelism type
	R) rgr_opt="${OPTARG}" ;; # Regridding options
	r) rgr_map="${OPTARG}" ;; # Regridding map
	S) yr_srt_prv="${OPTARG}" ;; # Start year previous
	s) yr_srt="${OPTARG}" ;; # Start year
	t) thr_usr="${OPTARG}" ;; # Thread number
	v) var_lst="${OPTARG}" ;; # Variables
	X) drc_xtn="${OPTARG}" ;; # Extended climo directory
	x) drc_prv="${OPTARG}" ;; # Previous climo directory
	Y) drc_rgr_xtn="${OPTARG}" ;; # Regridded extended climo directory
	y) drc_rgr_prv="${OPTARG}" ;; # Regridded previous climo directory
	-) LONG_OPTARG="${OPTARG#*=}"
	   case ${OPTARG} in
	       # Hereafter ${OPTARG} is long argument key, and ${LONG_OPTARG}, if any, is long argument value
	       # Long options with no argument, no short option counterpart
	       # Long options with argument, no short option counterpart
	       # Long options with short counterparts, ordered by short option key
	       dec_md=?* | december_mode=?* | dec_mode=?* ) dec_md="${LONG_OPTARG}" ;; # -a # December mode
	       clm_md=?* | climatology_mode=?* | climo_mode=?* ) clm_md="${LONG_OPTARG}" ;; # -C # Climatology mode
	       caseid=?* | case_id=?* | case=?* ) caseid="${LONG_OPTARG}" ;; # -c # CASEID
	       dbg_lvl=?* | dbg=?* | debug=?* | debug_level=?* ) dbg_lvl="${LONG_OPTARG}" ;; # -d # Debugging level
	       yr_end_prv=?* | prv_yr_end=?* | previous_end=?* ) yr_end_prv="${LONG_OPTARG}" ;; # -E # End year previous
	       yr_end=?* | end_yr=?* | year_end=?* | end_year=?* | end=?* ) yr_end="${LONG_OPTARG}" ;; # -e # End year
	       fml_nm=?* | family_name=?* | family=?* ) fml_nm="${LONG_OPTARG}" ;; # -f # Family name
	       hst_nm=?* | history_name=?* | history=?* ) hst_nm="${LONG_OPTARG}" ;; # -h # History tape name
	       drc_in=?* | in_drc=?* | dir_in=?* | in_dir=?* | input=?* ) drc_in="${LONG_OPTARG}" ;; # -i # Input directory
	       lnk_flg | link_flag | no_amwg_link | no_amwg_links | no_AMWG_link | no_AMWG_links ) lnk_flg='No' ;; # -l # Link ACME to AMWG name
	       lnk_flg=?* | link_flag=?* | no_amwg_link=?* | no_amwg_links=?* | no_AMWG_link=?* | no_AMWG_links=?* ) echo "No argument allowed for --${OPTARG switch}" >&2; exit 1 ;; # -l # Link ACME to AMWG name
	       mdl_nm=?* | model_name=?* | model=?* ) mdl_nm="${LONG_OPTARG}" ;; # -m # Model name
	       nco_opt=?* | nco=?* | nco_options=?* ) nco_opt="${LONG_OPTARG}" ;; # -n # NCO options
	       drc_out=?* | out_drc=?* | dir_out=?* | out_dir=?* | output=?* ) drc_out_usr="${LONG_OPTARG}" ;; # -o # Output directory
	       drc_rgr=?* | rgr_drc=?* | dir_regrid=?* | regrid_dir=?* | regrid=?* ) drc_rgr_usr="${LONG_OPTARG}" ;; # -O # Regridded directory
	       par_typ=?* | par_md=?* | parallel_type=?* | parallel_mode=?* | parallel=?* ) par_typ="${LONG_OPTARG}" ;; # -p # Parallelism type
	       rgr_opt=?* | regrid_options=?* ) rgr_opt="${LONG_OPTARG}" ;; # -R # Regridding options
	       rgr_map=?* | regrid_map=?* | map=?* ) rgr_map="${LONG_OPTARG}" ;; # -r # Regridding map
	       yr_srt_prv=?* | prv_yr_srt=?* | previous_start=?* ) yr_srt_prv="${LONG_OPTARG}" ;; # -S # Start year previous
	       yr_srt=?* | start_yr=?* | year_start=?* | start_year=?* | start=?* ) yr_srt="${LONG_OPTARG}" ;; # -s # Start year
	       thr_nbr=?* | thread_number=?* | thread=?* | threads=?* ) thr_usr="${LONG_OPTARG}" ;; # -t # Thread number
	       var_lst=?* | variable_list=?* | variable=?* | variables=?* ) var_lst="${LONG_OPTARG}" ;; # -v # Variables
	       drc_xtn=?* | xtn_drc=?* | extended_dir=?* | extended_climo=?* | extended=?* ) drc_xtn="${LONG_OPTARG}" ;; # -X # Extended climo directory
	       drc_prv=?* | prv_drc=?* | previous_dir=?* | previous_climo=?* | previous=?* ) drc_prv="${LONG_OPTARG}" ;; # -x # Previous climo directory
	       drc_rgr_xtn=?* | drc_xtn_rgr=?* | regridded_extended=?* | extended_regridded=?* ) drc_rgr_xtn="${LONG_OPTARG}" ;; # -Y # Regridded extended climo directory
	       drc_rgr_prv=?* | drc_prv_rgr=?* | regridded_previous=?* | previous_regridded=?* ) drc_rgr_prv="${LONG_OPTARG}" ;; # -y # Regridded previous climo directory
               '' ) break ;; # "--" terminates argument processing
               * ) printf "\nERROR: Illegal option ${fnt_bld}--${OPTARG}${fnt_nrm}" >&2; fnc_usg_prn ;;
	   esac ;;
	\?) # Unrecognized option
	    printf "\nERROR: Option ${fnt_bld}-${OPTARG}${fnt_nrm} not allowed" >&2
	    fnc_usg_prn ;;
    esac
done
shift $((OPTIND-1)) # Advance one argument

# Determine mode first (this helps determine other defaults)
if [ -n "${yr_srt_prv}" ]; then
    # Specifying only yr_srt_prv implies incremental method
    # Specifying both yr_srt_prv and yr_end_prv implies binary method
    xtn_flg='Yes'
    if [ -n "${yr_end_prv}" ]; then
	bnr_flg='Yes'
    else # !yr_end_prv binary method
	ncr_flg='Yes'
    fi # !yr_end_prv binary method
fi # !yr_srt_prv extended climo

# Derived variables
if [ -n "${drc_out_usr}" ]; then
    # Fancy %/ syntax removes trailing slash (e.g., from $TMPDIR)
    drc_out="${drc_out_usr%/}"
fi # !drc_out_usr
if [ -n "${drc_rgr_usr}" ]; then 
    drc_rgr="${drc_rgr_usr%/}"
else 
    drc_rgr="${drc_out%/}"
fi # !drc_rgr_usr
if [ -n "${drc_prv}" ]; then
    drc_prv="${drc_prv%/}"
else
    if [ "${bnr_flg}" = 'Yes' ]; then
	drc_prv="${drc_in}"
    fi # !bnr_flg
    if [ "${ncr_flg}" = 'Yes' ]; then
	drc_prv="${drc_out}"
    fi # !ncr_flg
fi # !drc_prv
if [ -n "${drc_xtn}" ]; then
    drc_xtn="${drc_xtn%/}"
else
    drc_xtn="${drc_prv}"
fi # !drc_xtn

# Doubly-derived variables
if [ -n "${drc_rgr_prv}" ]; then
    drc_rgr_prv="${drc_rgr_prv%/}"
else
    drc_rgr_prv="${drc_prv%/}"
fi # !drc_rgr_prv
if [ -n "${drc_rgr_xtn}" ]; then
    drc_rgr_xtn="${drc_rgr_xtn%/}"
else
    drc_rgr_xtn="${drc_xtn%/}"
fi # !drc_rgr_xtn

# Determine first full year
trim_leading_zeros ${yr_srt}
yr_srt_rth=${sng_trm}
yyyy_srt=`printf "%04d" ${yr_srt_rth}`
let yr_srtm1=${yr_srt_rth}-1
trim_leading_zeros ${yr_end}
yr_end_rth=${sng_trm}
yyyy_end=`printf "%04d" ${yr_end_rth}`
let yr_endm1=${yr_end_rth}-1
let yr_nbr=${yr_end_rth}-${yr_srt_rth}+1

# Derived variables
out_nm=${caseid}
if [ "${caseid}" = 'hist' ] || [ "${mdl_nm}" = 'mpaso' ] || [ "${mdl_nm}" = 'mpascice' ]; then
    mdl_typ='mpas'
fi # !caseid
if [ "${mdl_typ}" = 'mpas' ]; then
    out_nm="${mdl_nm}"
    hst_nm='hist'
fi # !mdl_typ
if [ -n "${fml_nm}" ]; then 
    out_nm="${fml_nm}"
fi # !fml_nm
# http://stackoverflow.com/questions/965053/extract-filename-and-extension-in-bash
# http://stackoverflow.com/questions/17420994/bash-regex-match-string
if [[ "${caseid}" =~ ^(.*)([0-9][0-9][0-9][0-9][01][0-9].nc.?)$ ]]; then
    mdl_typ='yyyymm'
    bs_nm="${BASH_REMATCH[1]}"
    bs_nm="$(basename ${bs_nm})"
    bs_nm="${bs_nm%.*}"
    bs_nm="${bs_nm%_*}"
    out_nm=${bs_nm}
    bs_sfx="${caseid#*.}"
fi # !caseid
if [ "${clm_md}" != 'ann' ] && [ "${clm_md}" != 'dly' ] && [ "${clm_md}" != 'mth' ]; then 
    echo "ERROR: User-defined climatology mode is ${clm_md}. Valid options are 'ann', 'dly', or 'mth' (default)"
    exit 1
fi # !clm_md
if [ "${clm_md}" = 'ann' ]; then 
    clm_nbr=1
    dec_md='sdd'
elif [ "${clm_md}" = 'mth' ]; then 
    clm_nbr=17
fi # !clm_md

if [ -n "${gaa_sng_std}" ]; then
    if [ "${yr_nbr}" -gt 1 ] ; then
	yrs_avg_sng="${yr_srt}-${yr_end}"
    else
	yrs_avg_sng="${yr_srt}"
    fi # !yr_nbr
    gaa_sng="${gaa_sng_std} --gaa yrs_averaged=${yrs_avg_sng}"
    nco_opt="${nco_opt} ${gaa_sng}"
fi # !var_lst
if [ -n "${var_lst}" ]; then
    nco_opt="${nco_opt} -v ${var_lst}"
fi # !var_lst
if [ -n "${hdr_pad}" ]; then
    nco_opt="${nco_opt} --hdr_pad=${hdr_pad}"
fi # !hdr_pad
if [ "${par_typ}" = 'bck' ]; then 
    par_opt=' &'
    par_opt_cf=''
elif [ "${par_typ}" = 'mpi' ]; then 
    mpi_flg='Yes'
    par_opt=' &'
    par_opt_cf=''
    if [ -n "${UVCDAT_SETUP_PATH}" ]; then
	printf "${spt_nm}: UVCDAT has been initialized in the shell running this job, and MPI-mode parallelization of ${spt_nm} is requested. Unfortunately UVCDAT's environment and the MPI-mode of ${spt_nm} do not play well together. The Workflow group is working toward a solution. The current workarounds are 1) do not use MPI-mode when UVCDAT is loaded or 2) do not initialize UVCDAT when invoking MPI-mode.\n"
    fi # !UVCDAT_SETUP_PATH
fi # !par_typ
if [ -n "${rgr_map}" ]; then 
    if [ ! -f "${rgr_map}" ]; then
	echo "ERROR: Unable to find specified regrid map ${rgr_map}"
	echo "HINT: Supply the full path-name for the regridding map"
	exit 1
    fi # ! -f
    rgr_opt="${rgr_opt} --map ${rgr_map}"
fi # !rgr_map
if [ -n "${thr_usr}" ]; then 
    thr_nbr="${thr_usr}"
fi # !thr_usr
yyyy_clm_srt=${yyyy_srt}
yyyy_clm_end=${yyyy_end}
yyyy_clm_srt_dec=${yyyy_srt}
yyyy_clm_end_dec=${yyyy_end}
mm_ann_srt='01' # [idx] First month used in annual climatology
mm_ann_end='12' # [idx] Last  month used in annual climatology
mm_djf_srt='01' # [idx] First month used in DJF climatology
mm_djf_end='12' # [idx] Last  month used in DJF climatology
yr_cln=${yr_nbr} # [nbr] Calendar years in climatology
if [ ${dec_md} = 'scd' ]; then 
    yyyy_clm_srt_dec=`printf "%04d" ${yr_srtm1}`
    yyyy_clm_end_dec=`printf "%04d" ${yr_endm1}`
    mm_ann_srt='12'
    mm_ann_end='11'
    mm_djf_srt='12'
    mm_djf_end='02'
    let yr_cln=${yr_cln}+1
fi # !scd

if [ "${mpi_flg}" = 'Yes' ]; then
    if [ -n "${COBALT_NODEFILE}" ]; then 
	nd_fl="${COBALT_NODEFILE}"
    elif [ -n "${PBS_NODEFILE}" ]; then 
	nd_fl="${PBS_NODEFILE}"
    elif [ -n "${SLURM_NODELIST}" ]; then 
	# SLURM returns compressed lists (e.g., "nid00[076-078,559-567]")
	# Convert this to file with uncompressed list (like Cobalt, PBS)
	# http://www.ceci-hpc.be/slurm_faq.html#Q12
	nd_fl='ncclimo.slurm_nodelist'
	nd_lst=`scontrol show hostname ${SLURM_NODELIST}`
	echo ${nd_lst} > ${nd_fl}
    else
	echo "ERROR: MPI job unable to find node list"
	echo "HINT: ${spt_nm} uses first node list found in \$COBALT_NODEFILE (= \"${COBALT_NODEFILE}\"), \$PBS_NODEFILE (= \"${PBS_NODEFILE}\"), \$SLURM_NODELIST (= \"${SLURM_NODELIST}\")"
	exit 1
    fi # !PBS
    if [ -n "${nd_fl}" ]; then 
	# NB: nodes are 0-based, e.g., [0..11]
	nd_idx=0
	for nd in `cat ${nd_fl} | uniq` ; do
	    nd_nm[${nd_idx}]=${nd}
	    let nd_idx=${nd_idx}+1
	done # !nd
	nd_nbr=${#nd_nm[@]}
	for ((clm_idx=1;clm_idx<=17;clm_idx++)); do
	    case "${HOSTNAME}" in 
		# 20160502: Remove tasks-per-node limits (ntasks, npernode) so round-robin algorithm can schedule multiple jobs on same node
		constance* | cori* | edison* | nid* | node* )
		    # 20160502: Non-interactive batch jobs at NERSC return HOSTNAME as nid*, not cori* or edison*
		    # 20160803: Non-interactive batch jobs at PNNL return HOSTNAME as node*, not constance*
		    # NB: NERSC staff says srun automatically assigns to unique nodes even without "-L $node" argument?
 		    cmd_mpi[${clm_idx}]="srun --nodelist ${nd_nm[$(((${clm_idx}-1) % ${nd_nbr}))]} --nodes=1" ; ;; # SLURM
# 		    cmd_mpi[${clm_idx}]="srun --nodelist ${nd_nm[$(((${clm_idx}-1) % ${nd_nbr}))]} --nodes=1 --ntasks=1" ; ;; # SLURM
		hopper* )
		    # NB: NERSC migrated from aprun to srun in 201601. Hopper commands will soon be deprecated.
		    cmd_mpi[${clm_idx}]="aprun -L ${nd_nm[$(((${clm_idx}-1) % ${nd_nbr}))]} -n 1" ; ;; # NERSC
		* )
		    cmd_mpi[${clm_idx}]="mpirun -H ${nd_nm[$(((${clm_idx}-1) % ${nd_nbr}))]} -n 1" ; ;; # Other (Cobalt, PBS)
#		    cmd_mpi[${clm_idx}]="mpirun -H ${nd_nm[$(((${clm_idx}-1) % ${nd_nbr}))]} -npernode 1 -n 1" ; ;; # Other
	    esac # !HOSTNAME
	done # !clm_idx
	if [ -n "${SLURM_NODELIST}" ]; then 
	    /bin/rm -f ${nd_fl}
	fi # !SLURM
    else # !nd_fl
	mpi_flg='No'
	for ((clm_idx=1;clm_idx<=17;clm_idx++)); do
	    cmd_mpi[${clm_idx}]=''
	done # !clm_idx
    fi # !nd_fl
    if [ -z "${thr_usr}" ]; then 
	if [ -n "${PBS_NUM_PPN}" ]; then
#	NB: use export OMP_NUM_THREADS when thr_nbr > 8
#	thr_nbr=${PBS_NUM_PPN}
	    thr_nbr=$((PBS_NUM_PPN > 8 ? 8 : PBS_NUM_PPN))
	fi # !pbs
    fi # !thr_usr
fi # !mpi

# Print initial state
if [ ${dbg_lvl} -ge 1 ]; then
    printf "dbg: bnr_flg  = ${bnr_flg}\n"
    printf "dbg: caseid   = ${caseid}\n"
    printf "dbg: cf_flg   = ${cf_flg}\n"
    printf "dbg: clm_md   = ${clm_md}\n"
    printf "dbg: dec_md   = ${dec_md}\n"
    printf "dbg: dbg_lvl  = ${dbg_lvl}\n"
    printf "dbg: drc_in   = ${drc_in}\n"
    printf "dbg: drc_nco  = ${drc_nco}\n"
    printf "dbg: drc_out  = ${drc_out}\n"
    printf "dbg: drc_prv  = ${drc_prv}\n"
    printf "dbg: drc_pwd  = ${drc_pwd}\n"
    printf "dbg: drc_rgr  = ${drc_rgr}\n"
    printf "dbg: drc_spt  = ${drc_spt}\n"
    printf "dbg: drc_xtn  = ${drc_xtn}\n"
    printf "dbg: fml_nm   = ${fml_nm}\n"
    printf "dbg: gaa_sng  = ${gaa_sng}\n"
    printf "dbg: hdr_pad  = ${hdr_pad}\n"
    printf "dbg: hst_nm   = ${hst_nm}\n"
    printf "dbg: lnk_flg  = ${lnk_flg}\n"
    printf "dbg: mdl_nm   = ${mdl_nm}\n"
    printf "dbg: mdl_typ  = ${mdl_typ}\n"
    printf "dbg: mpi_flg  = ${mpi_flg}\n"
    printf "dbg: nco_opt  = ${nco_opt}\n"
    printf "dbg: ncr_flg  = ${ncr_flg}\n"
    printf "dbg: nd_nbr   = ${nd_nbr}\n"
    printf "dbg: par_typ  = ${par_typ}\n"
    printf "dbg: rgr_map  = ${rgr_map}\n"
    printf "dbg: rgr_sfx  = ${rgr_sfx}\n"
    printf "dbg: thr_nbr  = ${thr_nbr}\n"
    printf "dbg: var_lst  = ${var_lst}\n"
    printf "dbg: xtn_flg  = ${xtn_flg}\n"
    printf "dbg: yyyy_end = ${yyyy_end}\n"
    printf "dbg: yyyy_srt = ${yyyy_srt}\n"
fi # !dbg
if [ ${dbg_lvl} -ge 2 ]; then
    printf "dbg: yyyy_srt   = ${yyyy_srt}\n"
    printf "dbg: yr_srt_rth = ${yr_srt_rth}\n"
    printf "dbg: yr_srtm1   = ${yr_srtm1}\n"
    printf "dbg: yr_endm1   = ${yr_endm1}\n"
    if [ ${mpi_flg} = 'Yes' ]; then
	for ((nd_idx=0;nd_idx<${nd_nbr};nd_idx++)); do
	    printf "dbg: nd_nm[${nd_idx}] = ${nd_nm[${nd_idx}]}\n"
	done # !nd
    fi # !mpi
fi # !dbg

# Create output directory
if [ -n "${drc_out}" ]; then 
    mkdir -p ${drc_out}
fi # !drc_out
if [ -n "${drc_rgr}" ]; then 
    mkdir -p ${drc_rgr}
fi # !drc_rgr

# Human-readable summary
date_srt=$(date +"%s")
if [ ${dbg_lvl} -ge 0 ]; then
    printf "Climatology generation invoked with command:\n"
    echo "${cmd_ln}"
fi # !dbg
printf "Started climatology generation for dataset ${caseid} at `date`.\n"
printf "Running climatology script ${spt_nm} from directory ${drc_spt}\n"
printf "NCO binaries version ${nco_vrs} from directory ${drc_nco}\n"
if [ "${xtn_flg}" = 'No' ]; then
    printf "Producing standard climatology from raw input files in directory ${drc_in}\n"
    printf "Output files to directory ${drc_out}\n"
fi # !xtn_flg
if [ "${bnr_flg}" = 'Yes' ]; then
    printf "Producing extended climatology in binary mode: Will combine pre-computed climatology in directory ${drc_prv} with pre-computed climatology in directory ${drc_in}\n"
    printf "Output files to directory ${drc_xtn}\n"
fi # !bnr_flg
if [ "${ncr_flg}" = 'Yes' ]; then
    printf "Producing extended climatology in incremental mode: Pre-computed climatology in directory ${drc_prv} will be incremented by raw input files in directory ${drc_in}\n"
    printf "Output files to directory ${drc_xtn}\n"
fi # !ncr_flg
#printf "Intermediate/temporary files written to directory ${drc_tmp}\n"
if [ "${bnr_flg}" = 'No' ]; then
    printf "Climatology from ${yr_nbr} years of contiguous raw data touching ${yr_cln} calendar years from YYYYMM = ${yyyy_clm_srt_dec}${mm_ann_srt} to ${yyyy_end}${mm_ann_end}.\n"
fi # !bnr_flg
if [ "${mdl_typ}" = 'yyyymm' ]; then
    printf "Filenames will be constructed with generic conventions as ${bs_nm}_YYYYMM.${bs_sfx}\n"
else # !mdl_typ
    printf "Filenames will be constructed with CESM'ish or ACME'ish conventions.\n"
fi # !mdl_typ
if [ ${clm_md} = 'ann' ]; then 
    printf "Each input file assumed to contain mean of one year.\n"
elif [ ${clm_md} = 'mth' ]; then 
    printf "Each input file assumed to contain mean of one month.\n"
fi # !mth
if [ ${clm_md} = 'mth' ]; then 
    if [ ${dec_md} = 'scd' ]; then 
	printf "Winter statistics based on seasonally contiguous December (scd-mode): DJF sequences are consecutive months that cross calendar-year boundaries.\n"
    else
	printf "Winter statistics based on seasonally discontiguous December (sdd-mode): DJF sequences comprise three months from the same calendar year.\n"
    fi # !scd
fi # !mth
if [ ${cf_flg} = 'Yes' ]; then 
    printf "Annotation for CF climatology attribute and climatology_bounds variable will be performed.\n"
else
    printf "Annotation for CF climatology attribute and climatology_bounds variable will not be performed.\n"
fi # !cf
if [ -n "${rgr_map}" ]; then 
    printf "This climatology will be regridded.\n"
else
    printf "This climatology will not be regridded.\n"
fi # !rgr

# Block 1: Generate, check, and store (but do not yet execute) commands

# Block 1 Loop 1: Climatologies based on monthly means
if [ "${clm_md}" = 'mth' ]; then
    clm_idx=0
    for mth in {01..12}; do
	let clm_idx=${clm_idx}+1
	MM=`printf "%02d" ${clm_idx}`
	yr_fl=''
	for yr in `seq ${yyyy_srt} ${yyyy_end}`; do
	    YYYY=`printf "%04d" ${yr}`
	    if [ ${mdl_typ} = 'cesm' ]; then
		yr_fl="${yr_fl} ${caseid}.${mdl_nm}.${hst_nm}.${YYYY}-${MM}.nc"
	    elif [ ${mdl_typ} = 'mpas' ]; then # Use MPAS not CESM conventions
		# 20161130: Old MPAS rule until today
		# yr_fl="${yr_fl} ${caseid}.${mdl_nm}.${YYYY}-${MM}-01_00.00.00.nc"
		# Example file: /scratch2/scratchdirs/golaz/ACME_simulations/20161117.beta0.A_WCYCL1850S.ne30_oEC_ICG.edison/run/mpascice.hist.am.timeSeriesStatsMonthly.0001-02-01.nc
 		yr_fl="${yr_fl} ${mdl_nm}.hist.am.timeSeriesStatsMonthly.${YYYY}-${MM}-01.nc"
	    elif [ ${mdl_typ} = 'yyyymm' ]; then # Generate from caseid + YYYYMM
		yr_fl="${yr_fl} ${bs_nm}_${YYYY}${MM}.${bs_sfx}"
	    fi # !cesm
	done # !yr
	if [ ${dec_md} = 'scd' ] && [ ${MM} = '12' ]; then 
	    yr_fl=''
	    for yr in `seq ${yr_srtm1} ${yr_endm1}`; do
		YYYY=`printf "%04d" ${yr}`
		if [ ${mdl_typ} = 'cesm' ]; then
		    yr_fl="${yr_fl} ${caseid}.${mdl_nm}.${hst_nm}.${YYYY}-${MM}.nc"
		elif [ ${mdl_typ} = 'mpas' ]; then # Use MPAS not CESM conventions
 		    yr_fl="${yr_fl} ${mdl_nm}.hist.am.timeSeriesStatsMonthly.${YYYY}-${MM}-01.nc"
		elif [ ${mdl_typ} = 'yyyymm' ]; then # Generate from caseid + YYYYMM
		    yr_fl="${yr_fl} ${bs_nm}_${YYYY}${MM}.${bs_sfx}"
		fi # !cesm
	    done # !yr
	    yyyy_clm_srt=${yyyy_clm_srt_dec}
	    yyyy_clm_end=${yyyy_clm_end_dec}
	fi # !scd
	# Check for existence of raw input only when file will be used
	if [ "${bnr_flg}" = 'No' ]; then
	    for fl_in in ${yr_fl} ; do
		if [ ! -f "${drc_in}/${fl_in}" ]; then
		    echo "ERROR: Unable to find required input file ${drc_in}/${fl_in}"
		    echo "HINT: All files implied to exist by the climatology bounds (start/end year/month) must be in ${drc_in} before ${spt_nm} will proceed"
		    exit 1
		fi # ! -f
	    done # !fl_in
	else # !bnr_flg
	    # In binary mode drc_out is actually used to locate input files from climatology B (same as output files in incremental mode)
	    drc_out="${drc_in}"
	fi # !bnr_flg
	fl_out[${clm_idx}]="${drc_out}/${out_nm}_${MM}_${yyyy_clm_srt}${MM}_${yyyy_clm_end}${MM}_climo.nc"
	cmd_clm[${clm_idx}]="${cmd_mpi[${clm_idx}]} ncra --cb -O ${nco_opt} -p ${drc_in} ${yr_fl} ${fl_out[${clm_idx}]}"
    done # !mth

    # Monthly output filenames constructed above; specify remaining (seasonal, annual) output names
    fl_out[13]="${drc_out}/${out_nm}_MAM_${yyyy_srt}03_${yyyy_end}05_climo.nc"
    fl_out[14]="${drc_out}/${out_nm}_JJA_${yyyy_srt}06_${yyyy_end}08_climo.nc"
    fl_out[15]="${drc_out}/${out_nm}_SON_${yyyy_srt}09_${yyyy_end}11_climo.nc"
    fl_out[16]="${drc_out}/${out_nm}_DJF_${yyyy_clm_srt_dec}${mm_djf_srt}_${yyyy_end}${mm_djf_end}_climo.nc"
    fl_out[17]="${drc_out}/${out_nm}_ANN_${yyyy_clm_srt_dec}${mm_ann_srt}_${yyyy_end}${mm_ann_end}_climo.nc"
    # Derive all seventeen regridded and AMWG names from output names
    for ((clm_idx=1;clm_idx<=17;clm_idx++)); do
	fl_amwg[${clm_idx}]=`expr match "${fl_out[${clm_idx}]}" '\(.*\)_.*_.*_climo.nc'` # Prune _YYYYYMM_YYYYMM_climo.nc
	fl_amwg[${clm_idx}]="${fl_amwg[${clm_idx}]}_climo.nc" # Replace with _climo.nc
	fl_amwg[${clm_idx}]="${fl_amwg[${clm_idx}]/${drc_out}\//}" # Delete prepended path to ease symlinking
	if [ -n "${rgr_map}" ]; then
	    fl_rgr[${clm_idx}]="${fl_out[${clm_idx}]/${drc_out}/${drc_rgr}}"
	    if [ "${drc_out}" = "${drc_rgr}" ]; then 
		# Append geometry suffix to regridded files in same directory as native climo
		# http://tldp.org/LDP/abs/html/string-manipulation.html
		dfl_sfx='rgr'
		rgr_sfx=`expr match "${rgr_map}" '.*_to_\(.*\).nc'`
		if [ "${#rgr_sfx}" -eq 0 ]; then
		    printf "${spt_nm}: WARNING Unable to extract geometric suffix from mapfile, will suffix regridded files with \"${dfl_sfx}\" instead\n"
		    rgr_sfx=${dfl_sfx}
		else
		    yyyymmdd_sng=`expr match "${rgr_sfx}" '.*\(\.[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]\)'` # Find YYYYYMMDD
		    if [ "${#yyyymmdd_sng}" -ne 0 ]; then
			rgr_sfx=${rgr_sfx%%${yyyymmdd_sng}} # Delete YYYYYMMDD
		    fi # !strlen
		fi # !strlen
		#    rgr_sfx=`expr match "${rgr_sfx}" '\(.*\)\.[0-9][0-9][0-9][0-9][0-9][0-9]'` # 
		fl_rgr[${clm_idx}]="${fl_rgr[${clm_idx}]/.nc/_${rgr_sfx}.nc}"
	    fi # !drc_rgr
	fi # !rgr_map
    done # !clm_idx
fi # !clm_md

# Block 1 Loop 2: Climatologies based on annual means
if [ "${clm_md}" = 'ann' ]; then
    clm_idx=1
    yr_fl=''
    for yr in `seq ${yyyy_srt} ${yyyy_end}`; do
	YYYY=`printf "%04d" ${yr}`
	yr_fl="${yr_fl} ${caseid}.${mdl_nm}.${hst_nm}.${YYYY}-${ann_sfx}.nc"
    done # !yr
    # Check for existence of raw input only when file will be used (NB: next ~12 lines duplicate monthly code)
    if [ "${bnr_flg}" = 'No' ]; then
	for fl_in in ${yr_fl} ; do
	    if [ ! -e "${drc_in}/${fl_in}" ]; then
		echo "ERROR: Unable to find required input file ${drc_in}/${fl_in}"
		echo "HINT: All files implied to exist by the climatology bounds (start/end year) must be in ${drc_in} before ${spt_nm} will proceed"
		exit 1
	    fi # ! -e
	done # !fl_in
    else # !bnr_flg
	# In binary mode drc_out is actually used to locate input files from climatology B (same as output files in incremental mode)
	drc_out="${drc_in}"
    fi # !bnr_flg
    fl_out[${clm_idx}]="${drc_out}/${out_nm}_ANN_${yyyy_srt}01_${yyyy_end}12_climo.nc"
    cmd_clm[${clm_idx}]="${cmd_mpi[${clm_idx}]} ncra -O ${nco_opt} -p ${drc_in} ${yr_fl} ${fl_out[${clm_idx}]} ${par_opt}"

    # Block 1 Loop 2: Climatological annual mean
    printf "Climatological annual mean...\n"
    if [ ${dbg_lvl} -ge 1 ]; then
	echo ${cmd_clm[${clm_idx}]}
    fi # !dbg
    if [ ${dbg_lvl} -le 1 ]; then
	eval ${cmd_clm[${clm_idx}]}
	if [ $? -ne 0 ]; then
	    printf "${spt_nm}: ERROR annual climo cmd_clm[${clm_idx}] failed\n"
	    exit 1
	fi # !err
    fi # !dbg
    wait
    
    # Block 2: Regrid climatological annual mean
    if [ -n "${rgr_map}" ]; then 
	printf "Regrid annual data...\n"
	cmd_rgr[${clm_idx}]="${cmd_mpi[${clm_idx}]} ncks -t ${thr_nbr} -O ${nco_opt} ${rgr_opt} ${fl_out[${clm_idx}]} ${fl_out[${clm_idx}]/.nc/.rgr.nc}"
	if [ ${dbg_lvl} -ge 1 ]; then
	    echo ${cmd_rgr[${clm_idx}]} ${par_opt}
	fi # !dbg
	if [ ${dbg_lvl} -le 1 ]; then
	    eval ${cmd_rgr[${clm_idx}]} ${par_opt}
	    if [ $? -ne 0 ]; then
		printf "${spt_nm}: ERROR annual regrid cmd_rgr[${clm_idx}] failed\n"
		exit 1
	    fi # !err
	fi # !dbg
	wait
	printf "Done with regridding\n"
    fi # !rgr_map
    
fi # !clm_md

# Many subsequent blocks only executed for normal and incremental monthly climos, not for binary climos, or non-monthly climos
if [ "${clm_md}" = 'mth' ] && [ "${bnr_flg}" = 'No' ]; then
    
    # Block 1 Loop 2: Execute and/or echo monthly climatology commands
    printf "Generating climatology...\n"
    for ((clm_idx=1;clm_idx<=12;clm_idx++)); do
	printf "Climatological monthly mean for month ${clm_idx} ...\n"
	if [ ${dbg_lvl} -ge 1 ]; then
	    echo ${cmd_clm[${clm_idx}]}
	fi # !dbg
	if [ ${dbg_lvl} -le 1 ]; then
	    if [ -z "${par_opt}" ]; then
		eval ${cmd_clm[${clm_idx}]}
		if [ $? -ne 0 ]; then
		    printf "${spt_nm}: ERROR monthly climo cmd_clm[${clm_idx}] failed. Debug this:\n${cmd_clm[${clm_idx}]}\n"
		    exit 1
		fi # !err
	    else # !par_opt
		eval ${cmd_clm[${clm_idx}]} ${par_opt} # eval always returns 0 on backgrounded processes
		clm_pid[${clm_idx}]=$!
		# Potential alternatives to eval:
		#	eval "${cmd_clm[${clm_idx}]}" # borken
		#       ${cmd_clm[${clm_idx}]} # borken
		#       "${cmd_clm[${clm_idx}]}" # borken
		#	exec "${cmd_clm[${clm_idx}]}" # borken
		#	$(${cmd_clm[${clm_idx}]}) # borken
		#	$("${cmd_clm[${clm_idx}]}") # works (when & inside cmd quotes)
	    fi # !par_opt
	fi # !dbg
    done # !clm_idx
    if [ -n "${par_opt}" ]; then
	for ((clm_idx=1;clm_idx<=12;clm_idx++)); do
	    wait ${clm_pid[${clm_idx}]}
	    if [ $? -ne 0 ]; then
		printf "${spt_nm}: ERROR monthly climo cmd_clm[${clm_idx}] failed. Debug this:\n${cmd_clm[${clm_idx}]}\n"
		exit 1
	    fi # !err
	done # !clm_idx
    fi # !par_opt
    wait
    
    # Block 1: Loop 4: Regrid first twelve files. Load-balance by using idle nodes (nodes not used for seasonal climatologies).
    if [ -n "${rgr_map}" ]; then 
	printf "Regrid monthly data...\n"
	for ((clm_idx=1;clm_idx<=12;clm_idx++)); do
	    # NB: Months, seasons, files are 1-based ([1..12], [13..16], [1..17]), nodes are 0-based ([0..11])
	    let nd_idx=$(((clm_idx-1+4) % nd_nbr))
	    if [ ${nd_idx} -lt 4 ]; then
		let nd_idx=${nd_idx}+4
	    fi # !nd
	    cmd_rgr[${clm_idx}]="${cmd_mpi[${nd_idx}]} ncks -t ${thr_nbr} -O ${nco_opt} ${rgr_opt} ${fl_out[${clm_idx}]} ${fl_rgr[${clm_idx}]}"
	    if [ "${mdl_typ}" = 'mpas' ]; then
		cmd_rgr[${clm_idx}]="${cmd_mpi[${nd_idx}]} ncremap -C -u .pid${spt_pid}.climo.${clm_idx}.tmp -P mpas -t ${thr_nbr} -m ${rgr_map} -i ${fl_out[${clm_idx}]} -o ${fl_rgr[${clm_idx}]}"
	    fi # !mdl_typ
	    if [ ${dbg_lvl} -ge 1 ]; then
		echo ${cmd_rgr[${clm_idx}]}
	    fi # !dbg
	    if [ ${dbg_lvl} -le 1 ]; then
		if [ -z "${par_opt}" ]; then
		    eval ${cmd_rgr[${clm_idx}]}
		    if [ $? -ne 0 ]; then
			printf "${spt_nm}: ERROR monthly regrid cmd_rgr[${clm_idx}] failed. Debug this:\n${cmd_rgr[${clm_idx}]}\n"
			exit 1
		    fi # !err
		else # !par_opt
		    eval ${cmd_rgr[${clm_idx}]} ${par_opt}
		    rgr_pid[${clm_idx}]=$!
		fi # !par_opt
	    fi # !dbg
	done 
	# Start seasonal means first, then wait() for monthly regridding to finish
    fi # !rgr_map
    
    # Block 2: Climatological seasonal means
    # Block 2 Loop 1: Generate seasonal commands
    printf "Climatological seasonal means...\n"
    cmd_clm[13]="${cmd_mpi[13]} ncra --cb -O -w 31,30,31 ${nco_opt} ${fl_out[3]} ${fl_out[4]} ${fl_out[5]} ${fl_out[13]}"
    cmd_clm[14]="${cmd_mpi[14]} ncra --cb -O -w 30,31,31 ${nco_opt} ${fl_out[6]} ${fl_out[7]} ${fl_out[8]} ${fl_out[14]}"
    cmd_clm[15]="${cmd_mpi[15]} ncra --cb -O -w 30,31,30 ${nco_opt} ${fl_out[9]} ${fl_out[10]} ${fl_out[11]} ${fl_out[15]}"
    cmd_clm[16]="${cmd_mpi[16]} ncra --cb -O -w 31,31,28 ${nco_opt} ${fl_out[12]} ${fl_out[1]} ${fl_out[2]} ${fl_out[16]}"

    # Block 2 Loop 2: Execute and/or echo seasonal climatology commands
    for ((clm_idx=13;clm_idx<=16;clm_idx++)); do
	if [ ${dbg_lvl} -ge 1 ]; then
	    echo ${cmd_clm[${clm_idx}]}
	fi # !dbg
	if [ ${dbg_lvl} -le 1 ]; then
	    if [ -z "${par_opt}" ]; then
		eval ${cmd_clm[${clm_idx}]}
		if [ $? -ne 0 ]; then
		    printf "${spt_nm}: ERROR seasonal climo cmd_clm[${clm_idx}] failed. Debug this:\n${cmd_clm[${clm_idx}]}\n"
		    exit 1
		fi # !err
	    else # !par_opt
		eval ${cmd_clm[${clm_idx}]} ${par_opt}
		clm_pid[${clm_idx}]=$!
	    fi # !par_opt
	fi # !dbg
    done # !clm_idx
    # wait() for monthly regridding, if any, to finish
    if [ -n "${rgr_map}" ]; then 
	if [ -n "${par_opt}" ]; then
	    for ((clm_idx=1;clm_idx<=12;clm_idx++)); do
		wait ${rgr_pid[${clm_idx}]}
		if [ $? -ne 0 ]; then
		    printf "${spt_nm}: ERROR monthly regrid cmd_rgr[${clm_idx}] failed. Debug this:\n${cmd_rgr[${clm_idx}]}\n"
		    exit 1
		fi # !err
	    done # !clm_idx
	fi # !par_opt
    fi # !rgr_map
    # wait() for seasonal climatologies to finish
    if [ -n "${par_opt}" ]; then
	for ((clm_idx=13;clm_idx<=16;clm_idx++)); do
	    wait ${clm_pid[${clm_idx}]}
	    if [ $? -ne 0 ]; then
		printf "${spt_nm}: ERROR seasonal climo cmd_clm[${clm_idx}] failed. Debug this:\n${cmd_clm[${clm_idx}]}\n"
		exit 1
	    fi # !err
	done # !clm_idx
    fi # !par_opt
    wait
    
    # Block 2: Loop 4: Regrid seasonal files. Load-balance by using idle nodes (nodes not used for annual mean).
    if [ -n "${rgr_map}" ]; then 
	printf "Regrid seasonal data...\n"
	for ((clm_idx=13;clm_idx<=16;clm_idx++)); do
	    let nd_idx=$(((clm_idx-1+4) % nd_nbr))
	    if [ ${nd_idx} -lt 4 ]; then
		let nd_idx=${nd_idx}+4
	    fi # !nd
	    cmd_rgr[${clm_idx}]="${cmd_mpi[${nd_idx}]} ncks -t ${thr_nbr} -O ${nco_opt} ${rgr_opt} ${fl_out[${clm_idx}]} ${fl_rgr[${clm_idx}]}"
	    if [ "${mdl_typ}" = 'mpas' ]; then
		cmd_rgr[${clm_idx}]="${cmd_mpi[${nd_idx}]} ncremap -C -u .pid${spt_pid}.climo.${clm_idx}.tmp -P mpas -t ${thr_nbr} -m ${rgr_map} -i ${fl_out[${clm_idx}]} -o ${fl_rgr[${clm_idx}]}"
	    fi # !mdl_typ
	    if [ ${dbg_lvl} -ge 1 ]; then
		echo ${cmd_rgr[${clm_idx}]}
	    fi # !dbg
	    if [ ${dbg_lvl} -le 1 ]; then
		if [ -z "${par_opt}" ]; then
		    eval ${cmd_rgr[${clm_idx}]}
		    if [ $? -ne 0 ]; then
			printf "${spt_nm}: ERROR seasonal regrid cmd_rgr[${clm_idx}] failed. Debug this:\n${cmd_rgr[${clm_idx}]}\n"
			exit 1
		    fi # !err
		else # !par_opt
		    eval ${cmd_rgr[${clm_idx}]} ${par_opt}
		    rgr_pid[${clm_idx}]=$!
		fi # !par_opt
	    fi # !dbg
	done 
	# Start annual mean first, then wait() for seasonal regridding to finish
    fi # !rgr_map
    
    # Block 3: Climatological annual mean (seventeenth file)
    printf "Climatological annual mean...\n"
    cmd_clm[17]="${cmd_mpi[17]} ncra --c2b -O -w 92,92,91,90 ${nco_opt} ${fl_out[13]} ${fl_out[14]} ${fl_out[15]} ${fl_out[16]} ${fl_out[17]}"
    if [ ${dbg_lvl} -ge 1 ]; then
	echo ${cmd_clm[17]}
    fi # !dbg
    if [ ${dbg_lvl} -le 1 ]; then
	if [ -z "${par_opt}" ]; then
	    eval ${cmd_clm[17]}
	    if [ $? -ne 0 ]; then
		printf "${spt_nm}: ERROR annual climo cmd_clm[17] failed. Debug this:\n${cmd_clm[17]}\n"
		exit 1
	    fi # !err
	else # !par_opt
	    eval ${cmd_clm[17]} ${par_opt}
	    clm_pid[17]=$!
	fi # !par_opt
    fi # !dbg
    # wait() for seasonal regridding, if any, to finish
    if [ -n "${rgr_map}" ]; then 
	if [ -n "${par_opt}" ]; then
	    for ((clm_idx=13;clm_idx<=16;clm_idx++)); do
		wait ${rgr_pid[${clm_idx}]}
		if [ $? -ne 0 ]; then
		    printf "${spt_nm}: ERROR seasonal regrid cmd_rgr[${clm_idx}] failed. Debug this:\n${cmd_rgr[${clm_idx}]}\n"
		    exit 1
		fi # !err
	    done # !clm_idx
	fi # !par_opt
    fi # !rgr_map
    # wait() for annual climatology to finish
    if [ -n "${par_opt}" ]; then
	wait ${clm_pid[17]}
	if [ $? -ne 0 ]; then
	    printf "${spt_nm}: ERROR annual climo cmd_clm[17] failed. Debug this:\n${cmd_clm[17]}\n"
	    exit 1
	fi # !err
    fi # !par_opt
    
    # Block 5: Regrid climatological annual mean
    if [ -n "${rgr_map}" ]; then 
	printf "Regrid annual data...\n"
	for ((clm_idx=17;clm_idx<=17;clm_idx++)); do
	    cmd_rgr[${clm_idx}]="${cmd_mpi[${clm_idx}]} ncks -t ${thr_nbr} -O ${nco_opt} ${rgr_opt} ${fl_out[${clm_idx}]} ${fl_rgr[${clm_idx}]}"
	    if [ "${mdl_typ}" = 'mpas' ]; then
		cmd_rgr[${clm_idx}]="${cmd_mpi[${clm_idx}]} ncremap -C -u .pid${spt_pid}.climo.${clm_idx}.tmp -P mpas -t ${thr_nbr} -m ${rgr_map} -i ${fl_out[${clm_idx}]} -o ${fl_rgr[${clm_idx}]}"
	    fi # !mdl_typ
	    if [ ${dbg_lvl} -ge 1 ]; then
		echo ${cmd_rgr[${clm_idx}]}
	    fi # !dbg
	    if [ ${dbg_lvl} -le 1 ]; then
		# NB: Do not background climatological mean regridding
		eval ${cmd_rgr[${clm_idx}]}
		if [ $? -ne 0 ]; then
		    printf "${spt_nm}: ERROR annual regrid cmd_rgr[${clm_idx}] failed. Debug this:\n${cmd_rgr[${clm_idx}]}\n"
		    exit 1
		fi # !err
	    fi # !dbg
	done 
    fi # !rgr_map
    
    # Link ACME-climo to AMWG-climo filenames
    # drc_pwd is always fully qualified path but drc_out and drc_rgr may be relative paths
    # Strategy: Start in drc_pwd, cd to drc_rgr, then link so return code comes from ln not cd
    if [ ${lnk_flg} = 'Yes' ]; then
	printf "Link ACME-climo to AMWG-climo filenames...\n"
	for ((clm_idx=1;clm_idx<=17;clm_idx++)); do
	    if [ -n "${rgr_map}" ]; then 
		cmd_lnk[${clm_idx}]="cd ${drc_pwd};cd ${drc_rgr};ln -s -f ${fl_rgr[${clm_idx}]/${drc_rgr}\//} ${fl_amwg[${clm_idx}]/${drc_rgr}\//}"
	    else
		cmd_lnk[${clm_idx}]="cd ${drc_pwd};cd ${drc_out};ln -s -f ${fl_out[${clm_idx}]/${drc_out}\//} ${fl_amwg[${clm_idx}]/${drc_out}\//}"
	    fi # !rgr_map
	    if [ ${dbg_lvl} -ge 1 ]; then
		echo ${cmd_lnk[${clm_idx}]}
	    fi # !dbg
	    if [ ${dbg_lvl} -le 1 ]; then
		eval ${cmd_lnk[${clm_idx}]}
		if [ $? -ne 0 ]; then
		    printf "${spt_nm}: ERROR linking ACME to AMWG filename cmd_lnk[${clm_idx}] failed. Debug this:\n${cmd_lnk[${clm_idx}]}\n"
		    exit 1
		fi # !err
	    fi # !dbg
	done # !clm_idx
	cd ${drc_pwd}
    fi # !lnk_flg
fi # !clm_md !bnr_flg

# Extended climos
if [ "${xtn_flg}" = 'Yes' ]; then
    mkdir -p ${drc_prv}
    mkdir -p ${drc_xtn}

    trim_leading_zeros ${yr_srt_prv}
    yr_srt_rth_prv=${sng_trm}
    yyyy_srt_prv=`printf "%04d" ${yr_srt_rth_prv}`
    yyyy_clm_srt_dec_prv=${yyyy_srt_prv}
    let yr_srtm1_prv=${yr_srt_rth_prv}-1
    if [ "${ncr_flg}" = 'Yes' ]; then
	let yr_end_prv=${yr_srt_rth}-1
    fi # !ncr_flg
    trim_leading_zeros ${yr_end_prv}
    yr_end_rth_prv=${sng_trm}
    yyyy_end_prv=`printf "%04d" ${yr_end_rth_prv}`
    let yr_endm1_prv=${yr_end_rth_prv}-1
    let yr_nbr_prv=${yr_end_rth_prv}-${yr_srt_rth_prv}+1
    let yr_nbr_xtn=${yr_nbr_prv}+${yr_nbr}

    wgt_prv=$(echo "${yr_nbr_prv}/${yr_nbr_xtn}" | bc -l)
    wgt_crr=$(echo "${yr_nbr}/${yr_nbr_xtn}" | bc -l)
    if [ "${bnr_flg}" = 'Yes' ]; then
	printf "Produce extended climatology as weighted average of two previously computed climatologies:\n"
    else # !bnr_flg
	printf "Produce extended climatology as weighted average of previously computed and incremental/new climatologies:\n"
    fi # !bnr_flg

    # Replace yr_srt by yr_srt_prv in "yrs_averaged" attribute
    nco_opt="${nco_opt/${yr_srt}-/${yr_srt_prv}-}"

    if [ "${clm_md}" = 'ann' ]; then
	printf "Previous/first climatology is ${yr_nbr_prv} years from ${yyyy_srt_prv} to ${yyyy_end_prv}, weight = ${wgt_prv}\n"
	printf "Current/second climatology is ${yr_nbr} years from ${yyyy_srt} to ${yyyy_end}, weight = ${wgt_crr}\n"
	printf "Extended climatology is ${yr_nbr_xtn} years from ${yyyy_srt_prv} to ${yyyy_end}\n"
    fi # !clm_md

    if [ "${clm_md}" = 'mth' ]; then

	printf "Previous/first climatology is ${yr_nbr_prv} years from ${yyyy_clm_srt_dec_prv}${mm_ann_srt} to ${yyyy_end_prv}${mm_ann_end}, weight = ${wgt_prv}\n"
	printf "Current/second climatology is ${yr_nbr} years from ${yyyy_clm_srt_dec}${mm_ann_srt} to ${yyyy_end}${mm_ann_end}, weight = ${wgt_crr}\n"
	printf "Extended climatology is ${yr_nbr_xtn} years from ${yyyy_clm_srt_dec_prv}${mm_ann_srt} to ${yyyy_end}${mm_ann_end}\n"
    
	clm_idx=0
	for mth in {01..12}; do
	    let clm_idx=${clm_idx}+1
	    MM=`printf "%02d" ${clm_idx}`
	    fl_prv[${clm_idx}]="${drc_prv}/${out_nm}_${MM}_${yyyy_srt_prv}${MM}_${yyyy_end_prv}${MM}_climo.nc"
	    fl_xtn[${clm_idx}]="${drc_xtn}/${out_nm}_${MM}_${yyyy_srt_prv}${MM}_${yyyy_end}${MM}_climo.nc"
	done # !mth
	if [ ${dec_md} = 'scd' ]; then 
	    yyyy_clm_srt_dec_prv=`printf "%04d" ${yr_srtm1_prv}`
	    yyyy_clm_end_dec_prv=`printf "%04d" ${yr_endm1_prv}`
	    clm_idx=12
	    MM=`printf "%02d" ${clm_idx}`
	    fl_prv[${clm_idx}]="${drc_prv}/${out_nm}_${MM}_${yyyy_clm_srt_dec_prv}${MM}_${yyyy_clm_end_dec_prv}${MM}_climo.nc"
	    fl_xtn[${clm_idx}]="${drc_xtn}/${out_nm}_${MM}_${yyyy_clm_srt_dec_prv}${MM}_${yyyy_clm_end_dec}${MM}_climo.nc"
	fi # !scd
	
	fl_prv[13]="${drc_prv}/${out_nm}_MAM_${yyyy_srt_prv}03_${yyyy_end_prv}05_climo.nc"
	fl_prv[14]="${drc_prv}/${out_nm}_JJA_${yyyy_srt_prv}06_${yyyy_end_prv}08_climo.nc"
	fl_prv[15]="${drc_prv}/${out_nm}_SON_${yyyy_srt_prv}09_${yyyy_end_prv}11_climo.nc"
	fl_prv[16]="${drc_prv}/${out_nm}_DJF_${yyyy_clm_srt_dec_prv}${mm_djf_srt}_${yyyy_end_prv}${mm_djf_end}_climo.nc"
	fl_prv[17]="${drc_prv}/${out_nm}_ANN_${yyyy_clm_srt_dec_prv}${mm_ann_srt}_${yyyy_end_prv}${mm_ann_end}_climo.nc"
	
	fl_xtn[13]="${drc_xtn}/${out_nm}_MAM_${yyyy_srt_prv}03_${yyyy_end}05_climo.nc"
	fl_xtn[14]="${drc_xtn}/${out_nm}_JJA_${yyyy_srt_prv}06_${yyyy_end}08_climo.nc"
	fl_xtn[15]="${drc_xtn}/${out_nm}_SON_${yyyy_srt_prv}09_${yyyy_end}11_climo.nc"
	fl_xtn[16]="${drc_xtn}/${out_nm}_DJF_${yyyy_clm_srt_dec_prv}${mm_djf_srt}_${yyyy_end}${mm_djf_end}_climo.nc"
	fl_xtn[17]="${drc_xtn}/${out_nm}_ANN_${yyyy_clm_srt_dec_prv}${mm_ann_srt}_${yyyy_end}${mm_ann_end}_climo.nc"
	
	# Derive all seventeen regridded and AMWG names from output names
	for ((clm_idx=1;clm_idx<=17;clm_idx++)); do
	    fl_rgr_prv[${clm_idx}]="${fl_rgr[${clm_idx}]/${drc_rgr}/${drc_rgr_prv}}"
	    fl_rgr_prv[${clm_idx}]="${fl_rgr_prv[${clm_idx}]/_${yyyy_srt}/_${yyyy_srt_prv}}"
	    fl_rgr_prv[${clm_idx}]="${fl_rgr_prv[${clm_idx}]/_${yyyy_end}/_${yyyy_end_prv}}"
	    
	    fl_rgr_xtn[${clm_idx}]="${fl_rgr[${clm_idx}]/${drc_rgr}/${drc_rgr_xtn}}"
	    fl_rgr_xtn[${clm_idx}]="${fl_rgr_xtn[${clm_idx}]/_${yyyy_srt}/_${yyyy_srt_prv}}"
	    
	    fl_amwg_xtn[${clm_idx}]=`expr match "${fl_xtn[${clm_idx}]}" '\(.*\)_.*_.*_climo.nc'` # Prune _YYYYYMM_YYYYMM_climo.nc
	    fl_amwg_xtn[${clm_idx}]="${fl_amwg[${clm_idx}]}_climo.nc" # Replace with _climo.nc
	    fl_amwg_xtn[${clm_idx}]="${fl_amwg[${clm_idx}]/${drc_xtn}\//}" # Delete prepended path to ease symlinking
	    if [ ${dec_md} = 'scd' ] ; then
		# Handle Dec, DJF, and ANN
		if [ ${clm_idx} -eq 12 ] || [ ${clm_idx} -eq 16 ] || [ ${clm_idx} -eq 17 ] ; then 
		    fl_rgr_prv[${clm_idx}]="${fl_rgr[${clm_idx}]/${drc_rgr}/${drc_rgr_prv}}"
		    fl_rgr_prv[${clm_idx}]="${fl_rgr_prv[${clm_idx}]/_${yyyy_clm_srt_dec}/_${yyyy_clm_srt_dec_prv}}"
		    if [ ${clm_idx} -eq 12 ] ; then 
			fl_rgr_prv[${clm_idx}]="${fl_rgr_prv[${clm_idx}]/_${yyyy_clm_end_dec}/_${yyyy_clm_end_dec_prv}}"
		    else
			fl_rgr_prv[${clm_idx}]="${fl_rgr_prv[${clm_idx}]/_${yyyy_end}/_${yyyy_end_prv}}"
		    fi # !Dec
		    
		    fl_rgr_xtn[${clm_idx}]="${fl_rgr[${clm_idx}]/${drc_rgr}/${drc_rgr_xtn}}"
		    fl_rgr_xtn[${clm_idx}]="${fl_rgr_xtn[${clm_idx}]/_${yyyy_clm_srt_dec}/_${yyyy_clm_srt_dec_prv}}"
		fi # !Dec, DJF, ANN
	    fi # !dec_md
	done # !clm_idx
	
	printf "Weight input climos to produce extended climo...\n"
	for ((clm_idx=1;clm_idx<=17;clm_idx++)); do
	    cmd_xtn[${clm_idx}]="${cmd_mpi[${clm_idx}]} ncflint -O ${nco_opt} -w ${wgt_prv},${wgt_crr} ${fl_prv[${clm_idx}]} ${fl_out[${clm_idx}]} ${fl_xtn[${clm_idx}]}"
	    if [ ${dbg_lvl} -ge 1 ]; then
		echo ${cmd_xtn[${clm_idx}]}
	    fi # !dbg
	    if [ ${dbg_lvl} -le 1 ]; then
		if [ -z "${par_opt}" ]; then
		    eval ${cmd_xtn[${clm_idx}]}
		    if [ $? -ne 0 ]; then
			printf "${spt_nm}: ERROR extended climo cmd_xtn[${clm_idx}] failed. Debug this:\n${cmd_xtn[${clm_idx}]}\n"
			exit 1
		    fi # !err
		else # !par_opt
		    eval ${cmd_xtn[${clm_idx}]} ${par_opt} # eval always returns 0 on backgrounded processes
		    xtn_pid[${clm_idx}]=$!
		fi # !par_opt
	    fi # !dbg
	done # !clm_idx
	if [ -n "${par_opt}" ]; then
	    for ((clm_idx=1;clm_idx<=17;clm_idx++)); do
		wait ${xtn_pid[${clm_idx}]}
		if [ $? -ne 0 ]; then
		    printf "${spt_nm}: ERROR extended climo cmd_xtn[${clm_idx}] failed. Debug this:\n${cmd_xtn[${clm_idx}]}\n"
		    exit 1
		fi # !err
	    done # !clm_idx
	fi # !par_opt
	wait
	
	if [ -n "${rgr_map}" ]; then 
	    printf "Weight input climos to produce extended regridded climo...\n"
	    for ((clm_idx=1;clm_idx<=17;clm_idx++)); do
		cmd_rgr_xtn[${clm_idx}]="${cmd_mpi[${clm_idx}]} ncflint -O ${nco_opt} -w ${wgt_prv},${wgt_crr} ${fl_rgr_prv[${clm_idx}]} ${fl_rgr[${clm_idx}]} ${fl_rgr_xtn[${clm_idx}]}"
		if [ ${dbg_lvl} -ge 1 ]; then
		    echo ${cmd_rgr_xtn[${clm_idx}]}
		fi # !dbg
		if [ ${dbg_lvl} -le 1 ]; then
		    if [ -z "${par_opt}" ]; then
			eval ${cmd_rgr_xtn[${clm_idx}]}
			if [ $? -ne 0 ]; then
			    printf "${spt_nm}: ERROR extended climo cmd_rgr_xtn[${clm_idx}] failed. Debug this:\n${cmd_rgr_xtn[${clm_idx}]}\n"
			    exit 1
			fi # !err
		    else # !par_opt
			eval ${cmd_rgr_xtn[${clm_idx}]} ${par_opt} # eval always returns 0 on backgrounded processes
			rgr_xtn_pid[${clm_idx}]=$!
		    fi # !par_opt
		fi # !dbg
	    done # !clm_idx
	    if [ -n "${par_opt}" ]; then
		for ((clm_idx=1;clm_idx<=17;clm_idx++)); do
		    wait ${rgr_xtn_pid[${clm_idx}]}
		    if [ $? -ne 0 ]; then
			printf "${spt_nm}: ERROR extended climo cmd_rgr_xtn[${clm_idx}] failed. Debug this:\n${cmd_rgr_xtn[${clm_idx}]}\n"
			exit 1
		    fi # !err
		done # !clm_idx
	    fi # !par_opt
	    wait
	fi # !rgr_map
	
	# Link ACME-climo to AMWG-climo filenames
	# drc_pwd is always fully qualified path but drc_out and drc_rgr may be relative paths
	# Strategy: Start in drc_pwd, cd to drc_rgr, then link so return code comes from ln not cd
	if [ ${lnk_flg} = 'Yes' ]; then
	    printf "Link extended ACME-climo to AMWG-climo filenames...\n"
	    for ((clm_idx=1;clm_idx<=17;clm_idx++)); do
		if [ -n "${rgr_map}" ]; then 
		    cmd_lnk_xtn[${clm_idx}]="cd ${drc_pwd};cd ${drc_rgr_xtn};ln -s -f ${fl_rgr_xtn[${clm_idx}]/${drc_rgr_xtn}\//} ${fl_amwg[${clm_idx}]/${drc_rgr_xtn}\//}"
		else
		    cmd_lnk_xtn[${clm_idx}]="cd ${drc_pwd};cd ${drc_xtn};ln -s -f ${fl_xtn[${clm_idx}]/${drc_xtn}\//} ${fl_amwg[${clm_idx}]/${drc_xtn}\//}"
		fi # !rgr_map
		if [ ${dbg_lvl} -ge 1 ]; then
		    echo ${cmd_lnk_xtn[${clm_idx}]}
		fi # !dbg
		if [ ${dbg_lvl} -le 1 ]; then
		    eval ${cmd_lnk_xtn[${clm_idx}]}
		    if [ $? -ne 0 ]; then
			printf "${spt_nm}: ERROR linking ACME to AMWG filename cmd_lnk_xtn[${clm_idx}] failed. Debug this:\n${cmd_lnk_xtn[${clm_idx}]}\n"
			exit 1
		    fi # !err
		fi # !dbg
	    done # !clm_idx
	    cd ${drc_pwd}
	fi # !lnk_flg
    fi # !clm_md
    
else # !xtn_flg extended climos
    
    yr_nbr_xtn=${yr_nbr}
    
fi # !xtn_flg extended climos

date_end=$(date +"%s")
printf "Completed ${yr_nbr_xtn}-year climatology generation for dataset ${caseid} at `date`.\n"
date_dff=$((date_end-date_srt))
echo "Quick plots of climatological annual mean:"
if [ -n "${yr_srt_prv}" ]; then
    if [ -n "${rgr_map}" ]; then 
	echo "ncview ${fl_rgr_xtn[${clm_nbr}]} &"
	echo "panoply ${fl_rgr_xtn[${clm_nbr}]} &"
    else
	echo "ncview ${fl_xtn[${clm_nbr}]} &"
	echo "panoply ${fl_xtn[${clm_nbr}]} &"
    fi # !rgr_map    
else
    if [ -n "${rgr_map}" ]; then 
	echo "ncview ${fl_rgr[${clm_nbr}]} &"
	echo "panoply ${fl_rgr[${clm_nbr}]} &"
    else
	echo "ncview ${fl_out[${clm_nbr}]} &"
	echo "panoply ${fl_out[${clm_nbr}]} &"
    fi # !rgr_map    
fi # !yr_srt_prv
echo "Elapsed time $((date_dff/60))m$((date_dff % 60))s"

# PMC: add SMB's Git (SHA1) hash info to climo files
# Assumes utility to add Git hash resides in ../utils/add_git_hash_to_netcdf_metadata
for ((clm_idx=1;clm_idx<=17;clm_idx++)); do
    fl_out_lst="${fl_out_lst} ${fl_out[${clm_idx}]}"
done
# CSZ: 20150826 disable until less fragile (than relative path) solution is found 
#cd ${drc_spt}
# ../utils/add_git_hash_to_netcdf_metadata ${fl_out_lst}

exit 0