{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# AM4 Performance\n", "\n", "\n", "* Wenchang Yang (wenchang@princeton.edu)\n", "* Department of Geosciences, Princeton University" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2018-11-27T17:04:24.285121Z", "start_time": "2018-11-27T17:04:23.632836Z" }, "code_folding": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Tue Nov 27 12:04:23 EST 2018\r\n" ] } ], "source": [ "# import\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "plt.rcParams['hatch.color']='g'\n", "import xarray as xr\n", "import pandas as pd\n", "import os, sys, glob, datetime\n", "from subprocess import check_output\n", "import math\n", "\n", "import geoxarray\n", "from geoplots import mapplot\n", "\n", "!date\n", "%matplotlib notebook" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2018-11-27T16:52:56.292583Z", "start_time": "2018-11-27T16:52:56.281036Z" }, "code_folding": [ 0 ] }, "outputs": [], "source": [ "def get_dataframe(cases=None):\n", "# if cases is None:\n", "# cases = glob.glob('run_FLOR_ctl_*') \n", " \n", " columns = ('expname', 'ntasks-per-node', 'tot_pes', 'tot_nodes', 'tot_spmy', 'tot_throughput', 'tot_cost')\n", "\n", " records = []\n", " for case in cases:\n", " slurm_logs = glob.glob(f'{case}/slurm-*')\n", " if slurm_logs:\n", " slurm_logs.sort()\n", " else:\n", " continue\n", " \n", " # ntasks-per-node\n", " s = check_output(f'grep ntasks-per-node {case}/test_AM4_*', shell=True) \\\n", " .decode('utf-8').strip().split('=')\n", " ntasks_per_node = int(s[-1])\n", " \n", "\n", " for slog in slurm_logs:\n", " # each slurm output has a single PE layout\n", " try:\n", " s = check_output(f'grep \"statistics\" {slog} -A326 |grep \"Total runtime\" -m 1', shell=True) \\\n", " .decode('utf-8').strip().split()\n", " except:\n", " continue # this is from a failed run\n", " tot_pes = int(s[-1]) - int(s[-2]) + 1\n", "# tot_nodes = tot_pes//ntasks_per_node\n", " tot_nodes = math.ceil(tot_pes/ntasks_per_node)\n", " \n", " \n", " # expname\n", " expname = check_output(f'grep expname {slog}', shell=True) \\\n", " .decode('utf-8').strip().split()[-1]\n", " \n", " \n", " # tot seconds per model years and throughput (model years per day)\n", " ss = check_output(f'grep \"statistics\" {slog} -A326 |grep \"Total runtime\"', shell=True) \\\n", " .decode('utf-8').strip().split('\\n')\n", " tot_spmy_list = []\n", " tot_throughput_list = []\n", " tot_cost_list = []\n", " for s in ss:\n", " spmy = float(s.split()[4])\n", " throughput = 24/(spmy/3600)\n", " tot_spmy_list.append(spmy)\n", " tot_throughput_list.append(throughput)\n", " cost = tot_nodes * 40 * (spmy/3600)\n", " tot_cost_list.append(cost)\n", " \n", " # create records\n", " for (tot_spmy,tot_throughput, tot_cost) in zip(tot_spmy_list, tot_throughput_list,tot_cost_list):\n", " \n", " record = (expname, ntasks_per_node, tot_pes, tot_nodes, tot_spmy, tot_throughput, tot_cost)\n", " records.append(record)\n", "\n", "\n", " df = pd.DataFrame(records, columns=columns)\n", " return df" ] }, { "cell_type": "markdown", "metadata": { "ExecuteTime": { "end_time": "2018-04-02T15:34:28.465428Z", "start_time": "2018-04-02T15:34:28.343191Z" } }, "source": [ "## FLOR_tiger2_intelmpi_18" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "ExecuteTime": { "end_time": "2018-11-29T16:05:38.735649Z", "start_time": "2018-11-29T16:05:33.904192Z" }, "code_folding": [], "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "DataFrame created for cases: ['288PE_36x', '288PE_40x', '384PE_36x', '384PE_40x', '432PE_36x', '432PE_40x', '576PE_36x', '576PE_39x', '576PE_40x', '768PE_36x', '768PE_40x', '864PE_36x', '864PE_40x']\n" ] }, { "data": { "text/html": [ "
| \n", " | expname | \n", "ntasks-per-node | \n", "tot_pes | \n", "tot_nodes | \n", "tot_spmy | \n", "tot_throughput | \n", "tot_cost | \n", "
|---|---|---|---|---|---|---|---|
| 0 | \n", "test_AM4_tigercpu_intelmpi_18_288PE_36x | \n", "36 | \n", "288 | \n", "8 | \n", "6892.295777 | \n", "12.535736 | \n", "612.648514 | \n", "
| 1 | \n", "test_AM4_tigercpu_intelmpi_18_288PE_40x | \n", "40 | \n", "288 | \n", "8 | \n", "7147.200836 | \n", "12.088649 | \n", "635.306741 | \n", "
| 2 | \n", "test_AM4_tigercpu_intelmpi_18_384PE_36x | \n", "36 | \n", "384 | \n", "11 | \n", "5539.359889 | \n", "15.597470 | \n", "677.032875 | \n", "
| 3 | \n", "test_AM4_tigercpu_intelmpi_18_384PE_36x | \n", "36 | \n", "384 | \n", "11 | \n", "5463.063406 | \n", "15.815302 | \n", "667.707750 | \n", "
| 4 | \n", "test_AM4_tigercpu_intelmpi_18_384PE_36x | \n", "36 | \n", "384 | \n", "11 | \n", "5425.061974 | \n", "15.926085 | \n", "663.063130 | \n", "
| 5 | \n", "test_AM4_tigercpu_intelmpi_18_384PE_40x | \n", "40 | \n", "384 | \n", "10 | \n", "5867.415356 | \n", "14.725394 | \n", "651.935040 | \n", "
| 6 | \n", "test_AM4_tigercpu_intelmpi_18_384PE_40x | \n", "40 | \n", "384 | \n", "10 | \n", "5711.345915 | \n", "15.127783 | \n", "634.593991 | \n", "
| 7 | \n", "test_AM4_tigercpu_intelmpi_18_384PE_40x | \n", "40 | \n", "384 | \n", "10 | \n", "5638.247302 | \n", "15.323911 | \n", "626.471922 | \n", "
| 8 | \n", "test_AM4_tigercpu_intelmpi_18_432PE_36x | \n", "36 | \n", "432 | \n", "12 | \n", "4907.505858 | \n", "17.605685 | \n", "654.334114 | \n", "
| 9 | \n", "test_AM4_tigercpu_intelmpi_18_432PE_36x | \n", "36 | \n", "432 | \n", "12 | \n", "4895.485138 | \n", "17.648915 | \n", "652.731352 | \n", "
| 10 | \n", "test_AM4_tigercpu_intelmpi_18_432PE_36x | \n", "36 | \n", "432 | \n", "12 | \n", "4815.065687 | \n", "17.943680 | \n", "642.008758 | \n", "
| 11 | \n", "test_AM4_tigercpu_intelmpi_18_432PE_36x | \n", "36 | \n", "432 | \n", "12 | \n", "4768.180008 | \n", "18.120121 | \n", "635.757334 | \n", "
| 12 | \n", "test_AM4_tigercpu_intelmpi_18_432PE_36x | \n", "36 | \n", "432 | \n", "12 | \n", "4945.333923 | \n", "17.471014 | \n", "659.377856 | \n", "
| 13 | \n", "test_AM4_tigercpu_intelmpi_18_432PE_36x | \n", "36 | \n", "432 | \n", "12 | \n", "4914.478885 | \n", "17.580704 | \n", "655.263851 | \n", "
| 14 | \n", "test_AM4_tigercpu_intelmpi_18_432PE_40x | \n", "40 | \n", "432 | \n", "11 | \n", "5318.315046 | \n", "16.245747 | \n", "650.016283 | \n", "
| 15 | \n", "test_AM4_tigercpu_intelmpi_18_432PE_40x | \n", "40 | \n", "432 | \n", "11 | \n", "5302.630319 | \n", "16.293800 | \n", "648.099261 | \n", "
| 16 | \n", "test_AM4_tigercpu_intelmpi_18_432PE_40x | \n", "40 | \n", "432 | \n", "11 | \n", "5301.272296 | \n", "16.297974 | \n", "647.933281 | \n", "
| 17 | \n", "test_AM4_tigercpu_intelmpi_18_432PE_40x | \n", "40 | \n", "432 | \n", "11 | \n", "5341.860761 | \n", "16.174139 | \n", "652.894093 | \n", "
| 18 | \n", "test_AM4_tigercpu_intelmpi_18_432PE_40x | \n", "40 | \n", "432 | \n", "11 | \n", "5221.829098 | \n", "16.545926 | \n", "638.223556 | \n", "
| 19 | \n", "test_AM4_tigercpu_intelmpi_18_576PE_36x | \n", "36 | \n", "576 | \n", "16 | \n", "3970.528853 | \n", "21.760325 | \n", "705.871796 | \n", "
| 20 | \n", "test_AM4_tigercpu_intelmpi_18_576PE_36x | \n", "36 | \n", "576 | \n", "16 | \n", "3900.488559 | \n", "22.151071 | \n", "693.420188 | \n", "
| 21 | \n", "test_AM4_tigercpu_intelmpi_18_576PE_36x | \n", "36 | \n", "576 | \n", "16 | \n", "3891.663093 | \n", "22.201305 | \n", "691.851217 | \n", "
| 22 | \n", "test_AM4_tigercpu_intelmpi_18_576PE_36x | \n", "36 | \n", "576 | \n", "16 | \n", "3883.847742 | \n", "22.245980 | \n", "690.461821 | \n", "
| 23 | \n", "test_AM4_tigercpu_intelmpi_18_576PE_36x | \n", "36 | \n", "576 | \n", "16 | \n", "3933.331196 | \n", "21.966114 | \n", "699.258879 | \n", "
| 24 | \n", "test_AM4_tigercpu_intelmpi_18_576PE_36x | \n", "36 | \n", "576 | \n", "16 | \n", "3856.227755 | \n", "22.405316 | \n", "685.551601 | \n", "
| 25 | \n", "test_AM4_tigercpu_intelmpi_18_576PE_39x | \n", "39 | \n", "576 | \n", "15 | \n", "4054.771572 | \n", "21.308229 | \n", "675.795262 | \n", "
| 26 | \n", "test_AM4_tigercpu_intelmpi_18_576PE_39x | \n", "39 | \n", "576 | \n", "15 | \n", "4268.870558 | \n", "20.239546 | \n", "711.478426 | \n", "
| 27 | \n", "test_AM4_tigercpu_intelmpi_18_576PE_39x | \n", "39 | \n", "576 | \n", "15 | \n", "3992.375320 | \n", "21.641252 | \n", "665.395887 | \n", "
| 28 | \n", "test_AM4_tigercpu_intelmpi_18_576PE_40x | \n", "40 | \n", "576 | \n", "15 | \n", "4072.259337 | \n", "21.216723 | \n", "678.709889 | \n", "
| 29 | \n", "test_AM4_tigercpu_intelmpi_18_576PE_40x | \n", "40 | \n", "576 | \n", "15 | \n", "3900.291361 | \n", "22.152191 | \n", "650.048560 | \n", "
| 30 | \n", "test_AM4_tigercpu_intelmpi_18_576PE_40x | \n", "40 | \n", "576 | \n", "15 | \n", "3796.825447 | \n", "22.755853 | \n", "632.804241 | \n", "
| 31 | \n", "test_AM4_tigercpu_intelmpi_18_576PE_40x | \n", "40 | \n", "576 | \n", "15 | \n", "4044.339026 | \n", "21.363194 | \n", "674.056504 | \n", "
| 32 | \n", "test_AM4_tigercpu_intelmpi_18_576PE_40x | \n", "40 | \n", "576 | \n", "15 | \n", "4062.517979 | \n", "21.267598 | \n", "677.086330 | \n", "
| 33 | \n", "test_AM4_tigercpu_intelmpi_18_576PE_40x | \n", "40 | \n", "576 | \n", "15 | \n", "4031.692894 | \n", "21.430204 | \n", "671.948816 | \n", "
| 34 | \n", "test_AM4_tigercpu_intelmpi_18_768PE_36x | \n", "36 | \n", "768 | \n", "22 | \n", "3279.457003 | \n", "26.345825 | \n", "801.645045 | \n", "
| 35 | \n", "test_AM4_tigercpu_intelmpi_18_768PE_36x | \n", "36 | \n", "768 | \n", "22 | \n", "3320.896034 | \n", "26.017075 | \n", "811.774586 | \n", "
| 36 | \n", "test_AM4_tigercpu_intelmpi_18_768PE_36x | \n", "36 | \n", "768 | \n", "22 | \n", "3295.352297 | \n", "26.218745 | \n", "805.530561 | \n", "
| 37 | \n", "test_AM4_tigercpu_intelmpi_18_768PE_40x | \n", "40 | \n", "768 | \n", "20 | \n", "3451.490829 | \n", "25.032661 | \n", "766.997962 | \n", "
| 38 | \n", "test_AM4_tigercpu_intelmpi_18_768PE_40x | \n", "40 | \n", "768 | \n", "20 | \n", "3325.071540 | \n", "25.984403 | \n", "738.904787 | \n", "
| 39 | \n", "test_AM4_tigercpu_intelmpi_18_768PE_40x | \n", "40 | \n", "768 | \n", "20 | \n", "3402.555242 | \n", "25.392681 | \n", "756.123387 | \n", "
| 40 | \n", "test_AM4_tigercpu_intelmpi_18_864PE_36x | \n", "36 | \n", "864 | \n", "24 | \n", "2953.215099 | \n", "29.256250 | \n", "787.524026 | \n", "
| 41 | \n", "test_AM4_tigercpu_intelmpi_18_864PE_36x | \n", "36 | \n", "864 | \n", "24 | \n", "2904.188434 | \n", "29.750136 | \n", "774.450249 | \n", "
| 42 | \n", "test_AM4_tigercpu_intelmpi_18_864PE_36x | \n", "36 | \n", "864 | \n", "24 | \n", "2903.213760 | \n", "29.760123 | \n", "774.190336 | \n", "
| 43 | \n", "test_AM4_tigercpu_intelmpi_18_864PE_40x | \n", "40 | \n", "864 | \n", "22 | \n", "3226.232065 | \n", "26.780467 | \n", "788.634505 | \n", "
| 44 | \n", "test_AM4_tigercpu_intelmpi_18_864PE_40x | \n", "40 | \n", "864 | \n", "22 | \n", "3134.483703 | \n", "27.564348 | \n", "766.207127 | \n", "
| 45 | \n", "test_AM4_tigercpu_intelmpi_18_864PE_40x | \n", "40 | \n", "864 | \n", "22 | \n", "3104.549054 | \n", "27.830129 | \n", "758.889769 | \n", "