athena.hh

//  $Id: athena.hh,v 1.5 2004/05/04 15:26:17 adams Exp $
//  Author:             Mark F. Adams
// Copyright (c) 2000 by Mark F. Adams 
//  Filename:           athena.hh
//
#ifndef  __ATHENA_H__
#define  __ATHENA_H__

#if defined(PROM_USE_IBM_LARGE_FILES)
#define _LARGE_FILES   // IBM
#endif

#include "prom_elem.hh"
#include "prom_options.hh"
#include "prom_perf.hh"

// hack for platforms w/o 'strncasecmp'
#if defined(PARCH_win32)
namespace {
  const signed char CASE_OFFSET = 'a' - 'A';
}
int strncasecmp(const char s1[], const char s2[], int n) 
{
  for (int p = 0; p < n; ++p) {
    char c1 = s1[p];
    char c2 = s2[p];
    if (c1 >= 'A' && c1 <= 'Z')
      c1 += CASE_OFFSET;
    if (c2 >= 'A' && c2 <= 'Z')
      c2 += CASE_OFFSET;
    if (c1 < c2)
      return -1;
    else if (c1 > c2)
      return 1;
    else if (c1 == 0 && c2 == 0)
      return 0;
  }
  return 0;
}
#endif

// class AthenaFile  (memory resident file or normal file)
//
class AthenaFile
{
public:
  AthenaFile( MPI_Comm comm, char *name, char *type, const int tag,
              const int factor = 1, AthenaFile *ifile = NULL ); 
  AthenaFile( char *name, char *type );
  ~AthenaFile();
  char *afgets( char *buff2, int bsz ); 
  int afprintf( const char *string ); // need to do all!!!
  int afprintf( char *string, char* );  
  int afprintf( char *string, int ); 
  int afprintf( char *string, int, int ); 
  int afprintf( char *string, int, double ); 
  int afprintf( char *string, double); 
  int afprintf( char *string, int, double,double,double); 
  int afprintf( char *string, int, int,double,double,double);
  int afprintf( char *string, int, int, int ); 
  int afprintf( char *string, int, int, int, int ); 
  int afprintf( char *string, int, int, int, int, int,int);
  int AdvanceFile( int &num, const int nIreal, const int coord_lsz );
  bool isForFEAP() const {return file_!=NULL; }
  // data
  FILE *file_;
  char *ptr_;
  char *base_;
  long int cnt_;
};

// class AthenaNode:
//
class AthenaNode : public PromStack, public PromMaterial
{ 
public:
  AthenaNode(int fid=-9, int gid=-9, int p=-9) { 
    setFeap(fid); index_.setGlobal(gid); ghost_.setProc( p ); }
  //virtual ~AthenaNode(){}
  AthenaNode &operator=( const AthenaNode &a ){
    assert(this!=NULL);
    ghost_ = a.ghost_;
    index_ = a.index_;
    coord_ = a.coord_;
    fIndex_ = a.fIndex_;
    return *this;
  }

  int  feap( int &ierr )const{ierr = !(this && fIndex_ >= 0); return fIndex_;} 
  int  feap()const{ assert(this!=NULL && fIndex_ >= 0); return fIndex_;} 
  void setFeap( int f ){ assert(this!=NULL); fIndex_ = f; }
  virtual int WriteFeap( AthenaFile *file ) const ;
  int  Read( int **buff );
  int  Write( int **buff )const;
  static int GetWrite_size( int &sz );
  bool isGhost()const{ int ierr; return (ghost_.proc(ierr) >= 0); }
  // data
public:
  int          fIndex_;
  PromGlobalID ghost_;
  PromLocalID  index_;
  PromCoord    coord_;
};

// class AthenaElem
//
class AthenaElem : public PromStack, public PromMaterial
{
protected:
  AthenaElem() : isghost_(FALSE), fid_(0), nNodes_(0), nodes_(NULL) {
    angles_[0] = angles_[1] = angles_[2] = angles_[3] = -9.0;
  }
  int   Init1(int feapid, int mat, int nen, int *nds, int *buff );
  int   Read( int nns, int *nds, int *&buff );
public:
  //virtual ~AthenaElem(){}
  int   Write( int *&buff );
  virtual int isErr()=0;
  int   GetWrite_size( int &sz ) const;
  int   WriteFeap(AthenaFile *file, const PromTable<int> &fid1_lid1, 
                  const PromTable<AthenaNode*> &fid1_ghost, 
                  const int lid1, const int nen_max, bool write_ang)const;
  int   numNodes()const{ return nNodes_; }  
  int   getFID()const{ assert(this!=NULL && fid_ != -9); return fid_; }
  int   getFID(int &ierr)const{
    assert(this!=NULL);ierr=(fid_==-9); return fid_;
  }
  void  setFID(int id ){assert(this!=NULL); fid_ = id;}
  bool  isGhost()const{ return isghost_; }
  void  setGhost( bool b = TRUE ){ isghost_ = b; }
  // data
protected:
  bool  isghost_;
  int   fid_;
  int   nNodes_;
public:
  int  *nodes_;
  float angles_[4];
};

// class AthenaElem8
//
class AthenaElem8 : public AthenaElem
{
public:
  AthenaElem8( int *&buff ){ 
    int ierr = Read( 8, nodes_private_, buff ); 
  }
  AthenaElem8( int fpid, int mat, int *buff ) {
    int ierr = Init1( fpid, mat, 8, nodes_private_, buff );  
  }
  virtual int isErr(){ 
    assert(this!=NULL); 
    return (nodes_private_ != nodes_ || nNodes_ != 8); 
  }
private:
  int nodes_private_[8];
};

// class AthenaElem4
//
class AthenaElem4 : public AthenaElem
{
public:
  AthenaElem4( int *&buff ){
    int ierr = Read( 4, nodes_private_, buff );
  }
  AthenaElem4( int fpid, int mat, int *buff ) {
    Init1( fpid, mat, 4, nodes_private_, buff );  
  }
  virtual int isErr(){ 
    assert(this!=NULL); 
    return (nodes_private_ != nodes_ || nNodes_ != 4); 
  }
private:
  int nodes_private_[4];
};

// class AthenaElemX
//
class AthenaElemX : public AthenaElem
{
public:
  AthenaElemX( int *&buff ) {
    nen_ = *buff; assert( nen_ > 0 );
    PetscMalloc(nen_*sizeof(int),&nodes_private_);
    Read( nen_, nodes_private_, buff );
  }
  AthenaElemX( int nen, int fpid, int mat, int *buff ) : nen_(nen) {
    assert( nen > 0 );
    PetscMalloc( nen*sizeof(int), &nodes_private_ );
    Init1( fpid, mat, nen, nodes_private_, buff );  
  }
  ~AthenaElemX(){ PetscFree(nodes_private_); }
  virtual int isErr(){ 
    assert(this!=NULL); 
    return (nodes_private_ != nodes_); 
  }
private:
  int nen_;
  int *nodes_private_;
};


class PromCommSpecs 
{
public:
  PromCommSpecs(double mfl = 100.0, double bd = 1.e+7, double lat = 1.e-5) {  
    latency_ = lat; bandwidth_ = bd; mflops_ = mfl;}
  int GetWeights(const int ndf, int &edgew, int &nodew, int &edgeconst );
  void print() const {
    PetscPrintf( MPI_COMM_WORLD,
                 "PromCommSpecs<mflops=%e,band width=%e,latentcy=%e>\n",
                 mflops_, bandwidth_, latency_);
  }
private:
  double latency_, bandwidth_, mflops_;
};

// class AthenaPartition 
//
class AthenaPartition : public PromContext
{
  friend class Athena;
public:
  AthenaPartition(const int myprocess, const int nprocess, const int nparts, 
                  const int sp, const int ep, const int inpgn, 
                  const int inpge, const PromComm &comm, 
                  PromOptions &opts, PromPerfMonitor &perf);
  AthenaPartition(const AthenaPartition *part1, 
                  const PromComm &comm, PromOptions &opts, PromPerfMonitor &perf );
  virtual ~AthenaPartition();

  //AthenaPartitioning 
  int Distribute( AthenaFile*, AthenaFile*, PromCommSpecs &commsp, int &nElems, 
                  const bool last );
protected:
  virtual int ReadIfile( AthenaFile* Ifile, int &nTotalNd, int &nElems, 
                         PromList<AthenaElem*> &elems,
                         AthenaNode **&inputGhosts ) = 0; 
  virtual int Write(AthenaFile *Ifile,  AthenaFile *Ofile, const int nIghost,
                    const PromTable<int> &fnodes, 
                    const PromTable<AthenaNode*> &ghosts, 
                    PromList<AthenaElem*> &elems) = 0;
  int Init(const int startp, const int stepp);
  int MakeDummy( const AthenaPartition *part1 );
  int Reorder( const PromList<AthenaElem*> &elems, 
               const PromTable<int> &fid1_lid1, 
               const PromTable<AthenaNode*> &fid1_ghost );
  int Partition( PromCommSpecs &commsp, const int nIreal, 
                 const PromList<AthenaElem*> &elems, int *&out );
  int CollectData( const int maxloc, const int fnode_part[],
                   PromList<AthenaElem*> &elems,
                   PromTable<int> &fnodes,
                   PromTable<AthenaNode*> &fghost,
                   const bool last );
  int CollectNewNodes(const int maxloc, const int fnode_part[], 
                      PromTable<int> &fnodes );
  int CollectNodeData( const int fnode_part[], 
                       const PromTable<int> &fnodes, int forig_gnode[] );
  int Send_RecvElemData( const AthenaNode orn[], const int forig_gnode[], 
                         PromList<AthenaElem*> &elems,
                         const PromTable<int> &fnodes, 
                         PromTable<AthenaNode*> &fid1_ghost,
                         const int fnode_part[] );
  int ReadAElem( int *buf, int &sz, const PromTable<int> &fid1_lid1, 
                 PromTable<AthenaNode*> &fid1_ghost,
                 PromTable<AthenaElem*> &felid1_elem, 
                 PromList<AthenaElem*> &elems,
                 PromTable<PromList<int*>*> &proc_list );
  int GetGhostTable( const PromTable<int> &fid1_lid1, 
                     PromTable<AthenaNode*> &fid1_ghost,  
                     AthenaNode **inputGhosts,
                     PromList<AthenaElem*> &elems);
  int CollectElemData(const int nIelems,PromList<AthenaElem*> &elems);
  int PickRealElems(const PromTable<int> &fid1_lid1,
                    const PromTable<AthenaNode*> &fid1_ghost,
                    const int nI, PromList<AthenaElem*> &relems,
                    PromList<AthenaElem*> &elems );
  int CalcOrder( const PromList<AthenaElem*> &elems,
                 const PromTable<int> &fid1_, int *old_new);
  int LocalReorder( const PromTable<int> &fid1_lid1, 
                    const PromTable<AthenaNode*> &fid1_ghost, 
                    const PromList<AthenaElem*> &elems,
                    const int old_new[] );
  int BfsOrder(PromTable<AthenaNode*> &list1,
               const PromTable<AthenaNode*> adjacs[], int old_new[] );
  int AthenaPartitionOrder( const int nblocs, 
                            PromTable<AthenaNode*> &list1, 
                            const PromTable<AthenaNode*> adjacs[], 
                            int old_new[], const int blockIS_flag );
  int AddRing( const PromTable<AthenaNode*> &list4,
               PromList<AthenaNode*> &list3, 
               const PromTable<AthenaNode*> adjacs[], 
               AthenaNode *&curr, int &ndone, int old_new[] );
  // data
  const int nCommprocs_;
  const int myCommproc_;
  const int nparts_;
  MPI_Comm  MPIComm_;
  MPI_Group Group_;
  MPI_Group mpi_group_world_;
public:
  const int nIghostNodes_;
  const int nIghostElems_;
  int   nInternalElems_;
  int   nOghostElems_;
  int   nLocalNd_;
  int   nOGhostNodes_;
  int   nforig_;
  int   nBlocks_;
  short int nmat_;
  short int nen_;
  short int ndf_;
  AthenaNode *nodes_;
  AthenaNode **ghosts_;
  int *part_gnode_;
  int *fnode_part_;
  int *nodeFIDs_;
  int *ghostFIDs_;
  int *bjacobi_blocks_;
  const PromComm &Comm_;
  PromOptions &options_;
  PromPerfMonitor &perf_mon_;
};


class Athena : public PromContext
{
public: 
  Athena( const PromComm &comm, PromOptions &opts, 
          PromPerfMonitor &perf );
  ~Athena();
  virtual int newPartition(const int myprocess, const int nprocess, 
                           const int nparts, const int startp, const int stepp,
                           const int inpgn, const int inpge, 
                           const PromComm &comm, PromOptions &opts, 
                           PromPerfMonitor &perf, AthenaPartition **pout) = 0;
  virtual int newPartition( const AthenaPartition *part1, const PromComm &comm, 
                            PromOptions &opts, PromPerfMonitor &perf, 
                            AthenaPartition **pout) = 0;
  int Distribute( char FEPname[256], int* ); // initial driver
  int PostRun();          // diagnostics
  static int MakeMaps( AthenaPartition *part1, AthenaPartition *part2, 
                       int *&pnode_fnode, int *&ghost_gnode, 
                       int &npLoc, int &npGst, MPI_Comm comm, const int myaddrsp,
                       const int numaddrsp, const int mylocalproc,
                       const int numlocalprocs );  
protected:
  static int GetGhostMaps( const int myAnode0, const int numnd, 
                           const int numndTOT, const int my0,
                           const int pnode_fnode[], const int nforg, 
                           const int fnode_addrsp[], const int nfaddrsp, 
                           const int anode_lproc[], const int naLoc, 
                           const int anode_fnode[], const int npGst, 
                           int ghost_gnode[],
                           const int myaddrsp, const int numaddrsp,
                           const int mylocalproc, const int numlocalprocs );
public:
  // partition data_
  int myAddrsp() const { return myproc_/proc_asp_; }
  int myLocalProc() const { return myproc_%proc_asp_; }
  int numAddrsp() const {
    int ii = numprocs_/proc_asp_; assert(numprocs_ % proc_asp_ == 0);
    //return (numprocs_ % proc_asp_ == 0 ? ii : ii+1); 
    return ii;
  }
  int numLocalProcs() const { 
    int left = numprocs_ % proc_asp_, ii = numprocs_ / proc_asp_ ;
    return ( left == 0 || myproc_ >= ii * proc_asp_ ) ? proc_asp_ : left ; 
  }
  // local (part) default starts
  static int getFstart( const int nnodes, const int myp, const int np );
  static int defaultFidPart(const int nnodes, const int fid, const int np );
  static int getNewTag(MPI_Comm comm){ 
    int tag; PetscCommGetNewTag( comm, &tag );  
    return tag;
  }
  // data
  AthenaPartition *part1_;
  AthenaPartition *part2_;
  //
  int myproc_;
  int numprocs_;
  int   proc_asp_;
  const PromComm &Comm_; 
  PromOptions &options_;
  PromPerfMonitor &perf_mon_;
};

/************************************************************************
 * FEAP specific specialization of Athena...
 */
class AthenaPartitionFeap : public AthenaPartition
{
public:
  AthenaPartitionFeap(const int myprocess, const int nprocess,
                      const int nparts, const int startp, const int stepp,
                      const int inpgn, const int inpge,
                      const PromComm &comm, PromOptions &opts, PromPerfMonitor &perf ):
    AthenaPartition( myprocess,  nprocess, nparts, startp, stepp, 
                     inpgn, inpge, comm, opts, perf ){}
  AthenaPartitionFeap( const AthenaPartition *part1, const PromComm &comm, 
                       PromOptions &opts, PromPerfMonitor &perf ) :
    AthenaPartition( part1, comm, opts, perf ){}
protected:
  virtual int ReadIfile( AthenaFile *Ifile, int &nTotalNd, int &nElems, 
                         PromList<AthenaElem*> &elems,
                         AthenaNode **&inputGhosts );
  virtual int Write( AthenaFile *Ifile, AthenaFile *Ofile, const int nIghost,
                     const PromTable<int> &fnodes, 
                     const PromTable<AthenaNode*> &ghosts, 
                     PromList<AthenaElem*> &elems);
private:
  int getRegion( AthenaFile *Ifile );
};

class AthenaFeap : public Athena
{
public:
  AthenaFeap(const PromComm &comm, PromOptions &opts, 
             PromPerfMonitor &perf ) : Athena( comm, opts, perf ) {}
  int newPartition( const int myprocess, const int nprocess, 
                    const int nparts, const int startp, const int stepp, 
                    const int inpgn, const int inpge, 
                    const PromComm &comm, PromOptions &opts, 
                    PromPerfMonitor &perf, AthenaPartition **pout ){
    AthenaPartition *out;
    out = new AthenaPartitionFeap(myprocess,  nprocess,  nparts, startp, stepp,
                                  inpgn, inpge, comm, opts, perf );
    *pout = out;
    return 0;
  }
  int newPartition( const AthenaPartition *part1, const PromComm &comm, 
                    PromOptions &opts, PromPerfMonitor &perf, 
                    AthenaPartition **pout ){
    int ierr; AthenaPartition *out;
    out = new AthenaPartitionFeap( part1, comm, opts, perf );
    *pout = out;
    return 0;
  }
};

#endif //  __ATHENA_H__

---