{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# FLOR Performance\n", "\n", "\n", "* Wenchang Yang (wenchang@princeton.edu)\n", "* Department of Geosciences, Princeton University" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "ExecuteTime": { "end_time": "2018-05-01T17:29:23.227202Z", "start_time": "2018-05-01T17:29:22.673044Z" }, "code_folding": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Tue May 1 13:29:22 EDT 2018\r\n" ] } ], "source": [ "# import\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "plt.rcParams['hatch.color']='g'\n", "import xarray as xr\n", "import pandas as pd\n", "import os, sys, glob, datetime\n", "from subprocess import check_output\n", "\n", "import geoxarray\n", "from geoplots import mapplot\n", "\n", "!date\n", "%matplotlib notebook" ] }, { "cell_type": "code", "execution_count": 146, "metadata": { "ExecuteTime": { "end_time": "2018-05-03T14:10:48.012183Z", "start_time": "2018-05-03T14:10:47.997202Z" }, "code_folding": [ 0 ] }, "outputs": [], "source": [ "def get_dataframe(cases=None):\n", "# if cases is None:\n", "# cases = glob.glob('run_FLOR_ctl_*') \n", " \n", " comps = ('atm', 'ocn', 'lnd', 'ice')\n", " pes_names = [f'{comp}_pes' for comp in comps]\n", " nodes_names = [f'{comp}_nodes' for comp in comps]\n", " speeds_names = [f'{comp}_spmy' for comp in comps]#spmy: seconds per model year\n", " speeds2_names = [f'{comp}_throughput' for comp in comps]# model years per day\n", " columns = ('expname', 'ntasks-per-node', 'tot_pes', 'tot_nodes', 'tot_spmy', 'tot_throughput', 'tot_cost') \\\n", " + tuple(p for pair in zip(pes_names, nodes_names, speeds_names, speeds2_names) for p in pair)\n", "\n", "\n", " records = []\n", " for case in cases:\n", " slurm_logs = glob.glob(f'{case}/slurm-*')\n", " if slurm_logs:\n", " slurm_logs.sort()\n", " else:\n", " continue\n", " \n", " # ntasks-per-node\n", " s = check_output(f'grep ntasks-per-node {case}/run_FLOR_*', shell=True) \\\n", " .decode('utf-8').strip().split('=')\n", " ntasks_per_node = int(s[-1])\n", " \n", "\n", " for slog in slurm_logs:\n", " # each slurm output has a single PE layout\n", " try:\n", " s = check_output(f'grep \"statistics\" {slog} -A326 |grep \"Total runtime\" -m 1', shell=True) \\\n", " .decode('utf-8').strip().split()\n", " except:\n", " continue # this is from a failed run\n", " tot_pes = int(s[-1]) - int(s[-2]) + 1\n", " tot_nodes = tot_pes//ntasks_per_node\n", " \n", " s = check_output(f'grep \"statistics\" {slog} -A326 |grep \"Atmosphere \" -m 1', shell=True) \\\n", " .decode('utf-8').strip().split()\n", " atm_pes = int(s[-1]) - int(s[-2]) + 1\n", " atm_nodes = atm_pes//ntasks_per_node\n", " \n", " s = check_output(f'grep \"statistics\" {slog} -A326 |grep \"Ocean \" -m 1', shell=True) \\\n", " .decode('utf-8').strip().split()\n", " ocn_pes = int(s[-1]) - int(s[-2]) + 1\n", " ocn_nodes = ocn_pes//ntasks_per_node\n", " \n", " s = check_output(f'grep \"statistics\" {slog} -A326 |grep \"Land \" -m 1', shell=True) \\\n", " .decode('utf-8').strip().split()\n", " lnd_pes = int(s[-1]) - int(s[-2]) + 1\n", " lnd_nodes = lnd_pes//ntasks_per_node\n", " \n", " s = check_output(f'grep \"statistics\" {slog} -A326 |grep \"Ice \" -m 1', shell=True) \\\n", " .decode('utf-8').strip().split()\n", " ice_pes = int(s[-1]) - int(s[-2]) + 1\n", " ice_nodes = ice_pes//ntasks_per_node\n", " \n", " \n", " # expname\n", " expname = check_output(f'grep expname {slog}', shell=True) \\\n", " .decode('utf-8').strip().split()[-1]\n", " \n", " \n", " # tot seconds per model years and throughput (model years per day)\n", " ss = check_output(f'grep \"statistics\" {slog} -A326 |grep \"Total runtime\"', shell=True) \\\n", " .decode('utf-8').strip().split('\\n')\n", " tot_spmy_list = []\n", " tot_throughput_list = []\n", " tot_cost_list = []\n", " for s in ss:\n", " spmy = float(s.split()[4])\n", " throughput = 24/(spmy/3600)\n", " tot_spmy_list.append(spmy)\n", " tot_throughput_list.append(throughput)\n", " cost = tot_nodes * 40 * (spmy/3600)\n", " tot_cost_list.append(cost)\n", " \n", " # atm seconds per model years\n", " ss = check_output(f'grep \"statistics\" {slog} -A326 |grep \"Atmosphere \"', shell=True) \\\n", " .decode('utf-8').strip().split('\\n')\n", " atm_spmy_list = []\n", " atm_throughput_list = []\n", " for s in ss:\n", " spmy = float(s.split()[3])\n", " atm_spmy_list.append(spmy)\n", " throughput = 24/(spmy/3600)\n", " atm_throughput_list.append(throughput)\n", " \n", " # lnd seconds per model years\n", " ss = check_output(f'grep \"statistics\" {slog} -A326 |grep \"Land \"', shell=True) \\\n", " .decode('utf-8').strip().split('\\n')\n", " lnd_spmy_list = []\n", " lnd_throughput_list = []\n", " for s in ss:\n", " spmy = float(s.split()[3])\n", " lnd_spmy_list.append(spmy)\n", " throughput = 24/(spmy/3600)\n", " lnd_throughput_list.append(throughput)\n", " \n", " # ice seconds per model years\n", " ss = check_output(f'grep \"statistics\" {slog} -A326 |grep \"Ice \"', shell=True) \\\n", " .decode('utf-8').strip().split('\\n')\n", " ice_spmy_list = []\n", " ice_throughput_list = []\n", " for s in ss:\n", " spmy = float(s.split()[3])\n", " ice_spmy_list.append(spmy)\n", " throughput = 24/(spmy/3600)\n", " ice_throughput_list.append(throughput)\n", " \n", " # ocn seconds per model years\n", " ss = check_output(f'grep \"statistics\" {slog} -A326 |grep \"Ocean \"', shell=True) \\\n", " .decode('utf-8').strip().split('\\n')\n", " ocn_spmy_list = []\n", " ocn_throughput_list = []\n", " for s in ss:\n", " spmy = float(s.split()[3])\n", " ocn_spmy_list.append(spmy)\n", " throughput = 24/(spmy/3600)\n", " ocn_throughput_list.append(throughput)\n", " \n", " # create records\n", " for (tot_spmy,tot_throughput, tot_cost,\n", " atm_spmy, atm_throughput,\n", " ocn_spmy, ocn_throughput,\n", " lnd_spmy, lnd_throughput,\n", " ice_spmy, ice_throughput) in zip(\n", " tot_spmy_list, tot_throughput_list,tot_cost_list,\n", " atm_spmy_list, atm_throughput_list,\n", " ocn_spmy_list, ocn_throughput_list,\n", " lnd_spmy_list, lnd_throughput_list,\n", " ice_spmy_list, ice_throughput_list):\n", " \n", " record = (expname, ntasks_per_node, tot_pes, tot_nodes, tot_spmy, tot_throughput, tot_cost,\n", " atm_pes, atm_nodes, atm_spmy, atm_throughput,\n", " ocn_pes, ocn_nodes, ocn_spmy, ocn_throughput,\n", " lnd_pes, lnd_nodes, lnd_spmy, lnd_throughput,\n", " ice_pes, ice_nodes, ice_spmy, ice_throughput)\n", " records.append(record)\n", "\n", "\n", " df = pd.DataFrame(records, columns=columns)\n", " return df" ] }, { "cell_type": "markdown", "metadata": { "ExecuteTime": { "end_time": "2018-04-02T15:34:28.465428Z", "start_time": "2018-04-02T15:34:28.343191Z" } }, "source": [ "## FLOR_tiger2_intelmpi_18" ] }, { "cell_type": "code", "execution_count": 158, "metadata": { "ExecuteTime": { "end_time": "2018-05-04T13:48:22.055446Z", "start_time": "2018-05-04T13:48:15.041267Z" }, "code_folding": [], "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "DataFrame created for cases: ['run_FLOR_ctl_1860_13x32PE', 'run_FLOR_ctl_1860_13x36PE', 'run_FLOR_ctl_1860_13x40PE', 'run_FLOR_ctl_1860_16x32PE', 'run_FLOR_ctl_1860_16x36PE', 'run_FLOR_ctl_1860_16x40PE', 'run_FLOR_ctl_1860_19x32PE', 'run_FLOR_ctl_1860_19x36PE', 'run_FLOR_ctl_1860_19x40PE', 'run_FLOR_ctl_1860_22x32PE', 'run_FLOR_ctl_1860_22x36PE', 'run_FLOR_ctl_1860_22x40PE', 'run_FLOR_ctl_1860_25x32PE', 'run_FLOR_ctl_1860_25x36PE', 'run_FLOR_ctl_1860_25x40PE']\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
expnamentasks-per-nodetot_pestot_nodestot_spmytot_throughputtot_costatm_pesatm_nodesatm_spmy...ocn_spmyocn_throughputlnd_peslnd_nodeslnd_spmylnd_throughputice_pesice_nodesice_spmyice_throughput
0ctl_1860_13x32PE_tiger2_intelmpi_18_416PE324161310258.7218828.4221021481.815383384129351.143560...5405.83278315.98273638412469.248423184.1242203841261.7998611398.061397
1ctl_1860_13x36PE_tiger2_intelmpi_18_468PE36468139724.9678548.8843481404.717579432128893.460454...5137.90170916.81620343212428.672860201.5522984321260.3406951431.869487
2ctl_1860_13x40PE_tiger2_intelmpi_18_520PE405201316201.0086715.3330012340.1456974801213920.326765...5530.21142715.62327248012797.925200108.28082648012380.295671227.191647
3ctl_1860_16x32PE_tiger2_intelmpi_18_512PE32512168503.91301710.1600291511.806759480157743.722867...5779.01606514.95064248015388.028741222.6639194801558.4780921477.476386
4ctl_1860_16x32PE_tiger2_intelmpi_18_512PE32512168308.98679510.3983801477.153208480157555.827424...5256.49223116.43681748015384.783796224.5416804801558.2854451482.359790
5ctl_1860_16x36PE_tiger2_intelmpi_18_576PE36576168108.60512110.6553471441.529799540157407.969535...5120.97131816.87179954015350.040693246.8284455401555.7116881550.841540
6ctl_1860_16x36PE_tiger2_intelmpi_18_576PE36576167322.52013911.7992161301.781358540156632.723105...4400.11143119.63586654015347.476995248.6495545401554.5351641584.298894
7ctl_1860_16x40PE_tiger2_intelmpi_18_640PE406401613776.2288096.2716732449.1073446001511600.472852...5528.85967915.62709260015704.957469122.56058560015511.974672168.758348
8ctl_1860_16x40PE_tiger2_intelmpi_18_640PE406401613478.7587736.4100862396.2237826001511369.944716...5515.00987815.66633660015662.889438130.33847760015516.751729167.198279
9ctl_1860_19x32PE_tiger2_intelmpi_18_608PE32608198087.20411910.6835441707.298647576187408.004101...6379.86272813.54261157618339.521576254.4757275761854.7084161579.281696
10ctl_1860_19x32PE_tiger2_intelmpi_18_608PE32608197233.28433111.9447811527.026692576186588.844742...5242.73213616.47995757618324.143239266.5488275761854.5098071585.035882
11ctl_1860_19x36PE_tiger2_intelmpi_18_684PE36684197148.59185712.0862961509.147170648186533.356473...5611.29883715.39750564818298.607767289.3427756481855.2830611562.865703
12ctl_1860_19x36PE_tiger2_intelmpi_18_684PE36684196682.65445612.9289941410.782607648186077.195799...5109.89499116.90837164818297.721683290.2039226481854.5938721582.595204
13ctl_1860_19x36PE_tiger2_intelmpi_18_684PE36684196915.99719112.4927751460.043851648186313.853058...5154.99043416.76045864818298.765993289.1895406481854.9835801571.378219
14ctl_1860_19x40PE_tiger2_intelmpi_18_760PE407601912571.1466866.8728812653.9087457201810448.623218...5551.35256215.56377572018585.021802147.68680472018610.529047141.516608
15ctl_1860_22x32PE_tiger2_intelmpi_18_704PE32704227361.00891811.7375211799.357736672216450.071304...6360.92442113.58293167221289.864840298.0699566722150.6881101704.541755
16ctl_1860_22x32PE_tiger2_intelmpi_18_704PE32704227705.32465611.2130251883.523805672217119.951030...5427.06791315.92019967221297.843074290.0856446722154.3076041590.937431
17ctl_1860_22x32PE_tiger2_intelmpi_18_704PE32704226483.97624513.3251571584.971971672215904.613462...5484.09206515.75465967221282.420587305.9267066722151.6371841673.212854
18ctl_1860_22x36PE_tiger2_intelmpi_18_792PE36792226503.70463513.2847361589.794466756215622.019542...5698.65923615.16146175621256.077512337.3978427562150.5958091707.651319
19ctl_1860_22x36PE_tiger2_intelmpi_18_792PE36792226105.59602614.1509531492.479029756215556.176845...5101.89059416.93489975621261.836989329.9762977562151.5400131676.367447
20ctl_1860_22x36PE_tiger2_intelmpi_18_792PE36792226045.16458014.2924151477.706897756215496.264039...5100.79889316.93852375621261.469820330.4396667562153.3915531618.233506
21ctl_1860_22x40PE_tiger2_intelmpi_18_880PE408802213073.8732156.6086003195.8356758402110566.417801...5530.87997315.62138484021720.569677119.90512984021711.760194121.389199
22ctl_1860_25x32PE_tiger2_intelmpi_18_800PE32800257110.71684312.1506741975.199123768245837.857406...6469.57227213.35482476824261.249502330.7183347682450.1130991724.100120
23ctl_1860_25x32PE_tiger2_intelmpi_18_800PE32800258051.08471510.7314732236.412421768247321.929587...5256.07837516.43811176824295.001707292.8796617682453.5438311613.631270
24ctl_1860_25x32PE_tiger2_intelmpi_18_800PE32800256077.98822814.2152301688.330063768245272.417670...5384.88896116.04489976824253.398277340.9652237682450.3651981715.470274
25ctl_1860_25x36PE_tiger2_intelmpi_18_900PE36900256368.66061213.5664321769.072392864245310.129469...5678.19472915.21610486424236.931608364.6621948642450.3175821717.093639
26ctl_1860_25x36PE_tiger2_intelmpi_18_900PE36900255790.57571314.9207961608.493254864244983.944396...5154.76749916.76118386424232.671755371.3385848642450.8631581698.675493
27ctl_1860_25x36PE_tiger2_intelmpi_18_900PE36900257333.36513611.7817672037.045871864246825.182201...5117.24306816.88409186424240.262327359.6069398642451.2539081685.725116
28ctl_1860_25x40PE_tiger2_intelmpi_18_1000PE4010002512412.7068116.9606093447.974114960249765.666804...5470.66678415.79332296024717.701338120.38433796024827.654176104.391426
\n", "

29 rows × 23 columns

\n", "
" ], "text/plain": [ " expname ntasks-per-node tot_pes \\\n", "0 ctl_1860_13x32PE_tiger2_intelmpi_18_416PE 32 416 \n", "1 ctl_1860_13x36PE_tiger2_intelmpi_18_468PE 36 468 \n", "2 ctl_1860_13x40PE_tiger2_intelmpi_18_520PE 40 520 \n", "3 ctl_1860_16x32PE_tiger2_intelmpi_18_512PE 32 512 \n", "4 ctl_1860_16x32PE_tiger2_intelmpi_18_512PE 32 512 \n", "5 ctl_1860_16x36PE_tiger2_intelmpi_18_576PE 36 576 \n", "6 ctl_1860_16x36PE_tiger2_intelmpi_18_576PE 36 576 \n", "7 ctl_1860_16x40PE_tiger2_intelmpi_18_640PE 40 640 \n", "8 ctl_1860_16x40PE_tiger2_intelmpi_18_640PE 40 640 \n", "9 ctl_1860_19x32PE_tiger2_intelmpi_18_608PE 32 608 \n", "10 ctl_1860_19x32PE_tiger2_intelmpi_18_608PE 32 608 \n", "11 ctl_1860_19x36PE_tiger2_intelmpi_18_684PE 36 684 \n", "12 ctl_1860_19x36PE_tiger2_intelmpi_18_684PE 36 684 \n", "13 ctl_1860_19x36PE_tiger2_intelmpi_18_684PE 36 684 \n", "14 ctl_1860_19x40PE_tiger2_intelmpi_18_760PE 40 760 \n", "15 ctl_1860_22x32PE_tiger2_intelmpi_18_704PE 32 704 \n", "16 ctl_1860_22x32PE_tiger2_intelmpi_18_704PE 32 704 \n", "17 ctl_1860_22x32PE_tiger2_intelmpi_18_704PE 32 704 \n", "18 ctl_1860_22x36PE_tiger2_intelmpi_18_792PE 36 792 \n", "19 ctl_1860_22x36PE_tiger2_intelmpi_18_792PE 36 792 \n", "20 ctl_1860_22x36PE_tiger2_intelmpi_18_792PE 36 792 \n", "21 ctl_1860_22x40PE_tiger2_intelmpi_18_880PE 40 880 \n", "22 ctl_1860_25x32PE_tiger2_intelmpi_18_800PE 32 800 \n", "23 ctl_1860_25x32PE_tiger2_intelmpi_18_800PE 32 800 \n", "24 ctl_1860_25x32PE_tiger2_intelmpi_18_800PE 32 800 \n", "25 ctl_1860_25x36PE_tiger2_intelmpi_18_900PE 36 900 \n", "26 ctl_1860_25x36PE_tiger2_intelmpi_18_900PE 36 900 \n", "27 ctl_1860_25x36PE_tiger2_intelmpi_18_900PE 36 900 \n", "28 ctl_1860_25x40PE_tiger2_intelmpi_18_1000PE 40 1000 \n", "\n", " tot_nodes tot_spmy tot_throughput tot_cost atm_pes atm_nodes \\\n", "0 13 10258.721882 8.422102 1481.815383 384 12 \n", "1 13 9724.967854 8.884348 1404.717579 432 12 \n", "2 13 16201.008671 5.333001 2340.145697 480 12 \n", "3 16 8503.913017 10.160029 1511.806759 480 15 \n", "4 16 8308.986795 10.398380 1477.153208 480 15 \n", "5 16 8108.605121 10.655347 1441.529799 540 15 \n", "6 16 7322.520139 11.799216 1301.781358 540 15 \n", "7 16 13776.228809 6.271673 2449.107344 600 15 \n", "8 16 13478.758773 6.410086 2396.223782 600 15 \n", "9 19 8087.204119 10.683544 1707.298647 576 18 \n", "10 19 7233.284331 11.944781 1527.026692 576 18 \n", "11 19 7148.591857 12.086296 1509.147170 648 18 \n", "12 19 6682.654456 12.928994 1410.782607 648 18 \n", "13 19 6915.997191 12.492775 1460.043851 648 18 \n", "14 19 12571.146686 6.872881 2653.908745 720 18 \n", "15 22 7361.008918 11.737521 1799.357736 672 21 \n", "16 22 7705.324656 11.213025 1883.523805 672 21 \n", "17 22 6483.976245 13.325157 1584.971971 672 21 \n", "18 22 6503.704635 13.284736 1589.794466 756 21 \n", "19 22 6105.596026 14.150953 1492.479029 756 21 \n", "20 22 6045.164580 14.292415 1477.706897 756 21 \n", "21 22 13073.873215 6.608600 3195.835675 840 21 \n", "22 25 7110.716843 12.150674 1975.199123 768 24 \n", "23 25 8051.084715 10.731473 2236.412421 768 24 \n", "24 25 6077.988228 14.215230 1688.330063 768 24 \n", "25 25 6368.660612 13.566432 1769.072392 864 24 \n", "26 25 5790.575713 14.920796 1608.493254 864 24 \n", "27 25 7333.365136 11.781767 2037.045871 864 24 \n", "28 25 12412.706811 6.960609 3447.974114 960 24 \n", "\n", " atm_spmy ... ocn_spmy ocn_throughput lnd_pes \\\n", "0 9351.143560 ... 5405.832783 15.982736 384 \n", "1 8893.460454 ... 5137.901709 16.816203 432 \n", "2 13920.326765 ... 5530.211427 15.623272 480 \n", "3 7743.722867 ... 5779.016065 14.950642 480 \n", "4 7555.827424 ... 5256.492231 16.436817 480 \n", "5 7407.969535 ... 5120.971318 16.871799 540 \n", "6 6632.723105 ... 4400.111431 19.635866 540 \n", "7 11600.472852 ... 5528.859679 15.627092 600 \n", "8 11369.944716 ... 5515.009878 15.666336 600 \n", "9 7408.004101 ... 6379.862728 13.542611 576 \n", "10 6588.844742 ... 5242.732136 16.479957 576 \n", "11 6533.356473 ... 5611.298837 15.397505 648 \n", "12 6077.195799 ... 5109.894991 16.908371 648 \n", "13 6313.853058 ... 5154.990434 16.760458 648 \n", "14 10448.623218 ... 5551.352562 15.563775 720 \n", "15 6450.071304 ... 6360.924421 13.582931 672 \n", "16 7119.951030 ... 5427.067913 15.920199 672 \n", "17 5904.613462 ... 5484.092065 15.754659 672 \n", "18 5622.019542 ... 5698.659236 15.161461 756 \n", "19 5556.176845 ... 5101.890594 16.934899 756 \n", "20 5496.264039 ... 5100.798893 16.938523 756 \n", "21 10566.417801 ... 5530.879973 15.621384 840 \n", "22 5837.857406 ... 6469.572272 13.354824 768 \n", "23 7321.929587 ... 5256.078375 16.438111 768 \n", "24 5272.417670 ... 5384.888961 16.044899 768 \n", "25 5310.129469 ... 5678.194729 15.216104 864 \n", "26 4983.944396 ... 5154.767499 16.761183 864 \n", "27 6825.182201 ... 5117.243068 16.884091 864 \n", "28 9765.666804 ... 5470.666784 15.793322 960 \n", "\n", " lnd_nodes lnd_spmy lnd_throughput ice_pes ice_nodes ice_spmy \\\n", "0 12 469.248423 184.124220 384 12 61.799861 \n", "1 12 428.672860 201.552298 432 12 60.340695 \n", "2 12 797.925200 108.280826 480 12 380.295671 \n", "3 15 388.028741 222.663919 480 15 58.478092 \n", "4 15 384.783796 224.541680 480 15 58.285445 \n", "5 15 350.040693 246.828445 540 15 55.711688 \n", "6 15 347.476995 248.649554 540 15 54.535164 \n", "7 15 704.957469 122.560585 600 15 511.974672 \n", "8 15 662.889438 130.338477 600 15 516.751729 \n", "9 18 339.521576 254.475727 576 18 54.708416 \n", "10 18 324.143239 266.548827 576 18 54.509807 \n", "11 18 298.607767 289.342775 648 18 55.283061 \n", "12 18 297.721683 290.203922 648 18 54.593872 \n", "13 18 298.765993 289.189540 648 18 54.983580 \n", "14 18 585.021802 147.686804 720 18 610.529047 \n", "15 21 289.864840 298.069956 672 21 50.688110 \n", "16 21 297.843074 290.085644 672 21 54.307604 \n", "17 21 282.420587 305.926706 672 21 51.637184 \n", "18 21 256.077512 337.397842 756 21 50.595809 \n", "19 21 261.836989 329.976297 756 21 51.540013 \n", "20 21 261.469820 330.439666 756 21 53.391553 \n", "21 21 720.569677 119.905129 840 21 711.760194 \n", "22 24 261.249502 330.718334 768 24 50.113099 \n", "23 24 295.001707 292.879661 768 24 53.543831 \n", "24 24 253.398277 340.965223 768 24 50.365198 \n", "25 24 236.931608 364.662194 864 24 50.317582 \n", "26 24 232.671755 371.338584 864 24 50.863158 \n", "27 24 240.262327 359.606939 864 24 51.253908 \n", "28 24 717.701338 120.384337 960 24 827.654176 \n", "\n", " ice_throughput \n", "0 1398.061397 \n", "1 1431.869487 \n", "2 227.191647 \n", "3 1477.476386 \n", "4 1482.359790 \n", "5 1550.841540 \n", "6 1584.298894 \n", "7 168.758348 \n", "8 167.198279 \n", "9 1579.281696 \n", "10 1585.035882 \n", "11 1562.865703 \n", "12 1582.595204 \n", "13 1571.378219 \n", "14 141.516608 \n", "15 1704.541755 \n", "16 1590.937431 \n", "17 1673.212854 \n", "18 1707.651319 \n", "19 1676.367447 \n", "20 1618.233506 \n", "21 121.389199 \n", "22 1724.100120 \n", "23 1613.631270 \n", "24 1715.470274 \n", "25 1717.093639 \n", "26 1698.675493 \n", "27 1685.725116 \n", "28 104.391426 \n", "\n", "[29 rows x 23 columns]" ] }, "execution_count": 158, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# data\n", "cases = glob.glob('run_FLOR_ctl_1860_??x??PE')\n", "cases.sort()\n", "csv_file = 'wy_FLOR_PE_scale.csv'\n", "try:\n", " 1/0\n", " df = pd.read_csv(csv_file)\n", " print('DataFrame loaded from', csv_file)\n", "except:\n", " df = get_dataframe(cases)\n", " df.to_csv(csv_file)\n", " print('DataFrame created for cases:', cases)\n", "df" ] }, { "cell_type": "code", "execution_count": 159, "metadata": { "ExecuteTime": { "end_time": "2018-05-04T13:48:27.369336Z", "start_time": "2018-05-04T13:48:25.857546Z" }, "code_folding": [ 0 ], "scrolled": false }, "outputs": [ { "data": { "application/javascript": [ "/* Put everything inside the global mpl namespace */\n", "window.mpl = {};\n", "\n", "\n", "mpl.get_websocket_type = function() {\n", " if (typeof(WebSocket) !== 'undefined') {\n", " return WebSocket;\n", " } else if (typeof(MozWebSocket) !== 'undefined') {\n", " return MozWebSocket;\n", " } else {\n", " alert('Your browser does not have WebSocket support.' +\n", " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", " 'Firefox 4 and 5 are also supported but you ' +\n", " 'have to enable WebSockets in about:config.');\n", " };\n", "}\n", "\n", "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n", " this.id = figure_id;\n", "\n", " this.ws = websocket;\n", "\n", " this.supports_binary = (this.ws.binaryType != undefined);\n", "\n", " if (!this.supports_binary) {\n", " var warnings = document.getElementById(\"mpl-warnings\");\n", " if (warnings) {\n", " warnings.style.display = 'block';\n", " warnings.textContent = (\n", " \"This browser does not support binary websocket messages. \" +\n", " \"Performance may be slow.\");\n", " }\n", " }\n", "\n", " this.imageObj = new Image();\n", "\n", " this.context = undefined;\n", " this.message = undefined;\n", " this.canvas = undefined;\n", " this.rubberband_canvas = undefined;\n", " this.rubberband_context = undefined;\n", " this.format_dropdown = undefined;\n", "\n", " this.image_mode = 'full';\n", "\n", " this.root = $('
');\n", " this._root_extra_style(this.root)\n", " this.root.attr('style', 'display: inline-block');\n", "\n", " $(parent_element).append(this.root);\n", "\n", " this._init_header(this);\n", " this._init_canvas(this);\n", " this._init_toolbar(this);\n", "\n", " var fig = this;\n", "\n", " this.waiting = false;\n", "\n", " this.ws.onopen = function () {\n", " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n", " fig.send_message(\"send_image_mode\", {});\n", " if (mpl.ratio != 1) {\n", " fig.send_message(\"set_dpi_ratio\", {'dpi_ratio': mpl.ratio});\n", " }\n", " fig.send_message(\"refresh\", {});\n", " }\n", "\n", " this.imageObj.onload = function() {\n", " if (fig.image_mode == 'full') {\n", " // Full images could contain transparency (where diff images\n", " // almost always do), so we need to clear the canvas so that\n", " // there is no ghosting.\n", " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", " }\n", " fig.context.drawImage(fig.imageObj, 0, 0);\n", " };\n", "\n", " this.imageObj.onunload = function() {\n", " fig.ws.close();\n", " }\n", "\n", " this.ws.onmessage = this._make_on_message_function(this);\n", "\n", " this.ondownload = ondownload;\n", "}\n", "\n", "mpl.figure.prototype._init_header = function() {\n", " var titlebar = $(\n", " '
');\n", " var titletext = $(\n", " '
');\n", " titlebar.append(titletext)\n", " this.root.append(titlebar);\n", " this.header = titletext[0];\n", "}\n", "\n", "\n", "\n", "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n", "\n", "}\n", "\n", "\n", "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n", "\n", "}\n", "\n", "mpl.figure.prototype._init_canvas = function() {\n", " var fig = this;\n", "\n", " var canvas_div = $('
');\n", "\n", " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n", "\n", " function canvas_keyboard_event(event) {\n", " return fig.key_event(event, event['data']);\n", " }\n", "\n", " canvas_div.keydown('key_press', canvas_keyboard_event);\n", " canvas_div.keyup('key_release', canvas_keyboard_event);\n", " this.canvas_div = canvas_div\n", " this._canvas_extra_style(canvas_div)\n", " this.root.append(canvas_div);\n", "\n", " var canvas = $('');\n", " canvas.addClass('mpl-canvas');\n", " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n", "\n", " this.canvas = canvas[0];\n", " this.context = canvas[0].getContext(\"2d\");\n", "\n", " var backingStore = this.context.backingStorePixelRatio ||\n", "\tthis.context.webkitBackingStorePixelRatio ||\n", "\tthis.context.mozBackingStorePixelRatio ||\n", "\tthis.context.msBackingStorePixelRatio ||\n", "\tthis.context.oBackingStorePixelRatio ||\n", "\tthis.context.backingStorePixelRatio || 1;\n", "\n", " mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n", "\n", " var rubberband = $('');\n", " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n", "\n", " var pass_mouse_events = true;\n", "\n", " canvas_div.resizable({\n", " start: function(event, ui) {\n", " pass_mouse_events = false;\n", " },\n", " resize: function(event, ui) {\n", " fig.request_resize(ui.size.width, ui.size.height);\n", " },\n", " stop: function(event, ui) {\n", " pass_mouse_events = true;\n", " fig.request_resize(ui.size.width, ui.size.height);\n", " },\n", " });\n", "\n", " function mouse_event_fn(event) {\n", " if (pass_mouse_events)\n", " return fig.mouse_event(event, event['data']);\n", " }\n", "\n", " rubberband.mousedown('button_press', mouse_event_fn);\n", " rubberband.mouseup('button_release', mouse_event_fn);\n", " // Throttle sequential mouse events to 1 every 20ms.\n", " rubberband.mousemove('motion_notify', mouse_event_fn);\n", "\n", " rubberband.mouseenter('figure_enter', mouse_event_fn);\n", " rubberband.mouseleave('figure_leave', mouse_event_fn);\n", "\n", " canvas_div.on(\"wheel\", function (event) {\n", " event = event.originalEvent;\n", " event['data'] = 'scroll'\n", " if (event.deltaY < 0) {\n", " event.step = 1;\n", " } else {\n", " event.step = -1;\n", " }\n", " mouse_event_fn(event);\n", " });\n", "\n", " canvas_div.append(canvas);\n", " canvas_div.append(rubberband);\n", "\n", " this.rubberband = rubberband;\n", " this.rubberband_canvas = rubberband[0];\n", " this.rubberband_context = rubberband[0].getContext(\"2d\");\n", " this.rubberband_context.strokeStyle = \"#000000\";\n", "\n", " this._resize_canvas = function(width, height) {\n", " // Keep the size of the canvas, canvas container, and rubber band\n", " // canvas in synch.\n", " canvas_div.css('width', width)\n", " canvas_div.css('height', height)\n", "\n", " canvas.attr('width', width * mpl.ratio);\n", " canvas.attr('height', height * mpl.ratio);\n", " canvas.attr('style', 'width: ' + width + 'px; height: ' + height + 'px;');\n", "\n", " rubberband.attr('width', width);\n", " rubberband.attr('height', height);\n", " }\n", "\n", " // Set the figure to an initial 600x600px, this will subsequently be updated\n", " // upon first draw.\n", " this._resize_canvas(600, 600);\n", "\n", " // Disable right mouse context menu.\n", " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n", " return false;\n", " });\n", "\n", " function set_focus () {\n", " canvas.focus();\n", " canvas_div.focus();\n", " }\n", "\n", " window.setTimeout(set_focus, 100);\n", "}\n", "\n", "mpl.figure.prototype._init_toolbar = function() {\n", " var fig = this;\n", "\n", " var nav_element = $('
')\n", " nav_element.attr('style', 'width: 100%');\n", " this.root.append(nav_element);\n", "\n", " // Define a callback function for later on.\n", " function toolbar_event(event) {\n", " return fig.toolbar_button_onclick(event['data']);\n", " }\n", " function toolbar_mouse_event(event) {\n", " return fig.toolbar_button_onmouseover(event['data']);\n", " }\n", "\n", " for(var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " // put a spacer in here.\n", " continue;\n", " }\n", " var button = $('