Team.h

#ifndef TEAM_H_INCLUDED
#define TEAM_H_INCLUDED

#include <dash/dart/if/dart.h>

#include <dash/Init.h>
#include <dash/Types.h>
#include <dash/Exception.h>

#include <dash/util/Locality.h>

#include <dash/internal/Logging.h>

#include <list>
#include <unordered_map>
#include <iostream>
#include <memory>
#include <type_traits>
#include <utility>


namespace dash {

// Forward-declarations
class Team;
std::ostream & operator<<(
  std::ostream & os,
  const Team   & team);

class Team
{
  template <typename U>
    friend class Shared;
  template <typename U, class P, class GM, class Ptr, class Ref >
    friend class GlobIter;
  template <typename U>
    friend class GlobRef;
  friend std::ostream & operator<<(
    std::ostream & os,
    const Team & team);

public:

  typedef struct iterator
  {
    int val;

    iterator(int v) : val(v) {}
    iterator & operator+=(const iterator & rhs) {
      val += rhs.val;
      return *this;
    }

    inline iterator & operator++() {
      return operator+=(1);
    }

    inline int operator*() const {
      return val;
    }

    inline operator int() const {
      return val;
    }
  } iterator;

  inline iterator begin()
  {
    return {0};
  }

  inline iterator end()
  {
    return {static_cast<int>(size())};
  }

public:
  typedef struct Deallocator
  {
    typedef std::function<void(void)> dealloc_function;
    void             * object;
    dealloc_function   deallocator;
  } Deallocator;

private:
  Team(
    dart_team_t id,
    Team      * parent    = nullptr,
    size_t      pos       = 0,
    size_t      nsiblings = 0);

  bool get_group() const
  {
    if (dash::is_initialized() && _group != DART_GROUP_NULL) {
      DASH_LOG_DEBUG("Team.get_group()");
      dart_team_get_group(_dartid, &_group);
    }
    return _group != DART_GROUP_NULL;
  }

protected:
  Team(const Team & t) = default;

private:
  Team & operator=(const Team & t) = delete;

public:
  Team(Team && t)
  : _dartid(t._dartid),
    _myid(t._myid),
    _size(t._size),
    _parent(t._parent),
    _child(t._child),
    _position(t._position),
    _num_siblings(t._num_siblings),
    _group(t._group),
    _deallocs(std::move(t._deallocs))
  {
    t._parent = nullptr;
    t._group  = nullptr;
    t._dartid = DART_TEAM_NULL;
  }

  Team & operator=(Team && t)
  {
    if (this != &t) {
      // Free existing resources
      free();
      // Take ownership of data from source
      _deallocs = std::move(t._deallocs);
      std::swap(_parent,    t._parent);
      std::swap(_group,     t._group);
      std::swap(_dartid,    t._dartid);
      _position     = t._position;
      _num_siblings = t._num_siblings;
      _myid         = t._myid;
      _size         = t._size;
    }
    return *this;
  }

  ~Team()
  {
    DASH_LOG_DEBUG_VAR("Team.~Team()", this);

    // Do not register static Team instances as static variable _team might
    // not be initialized at the time of their instantiation, yet:
    if (DART_TEAM_NULL != _dartid &&
        DART_TEAM_ALL  != _dartid) {
      Team::unregister_team(this);
    }

    if (_group != DART_GROUP_NULL) {
      if (DART_OK != dart_group_destroy(&_group)) {
        DASH_LOG_ERROR("dash::Team d'tor", "Failed to destroy DART group!");
      }
    }

    if (_child) {
      delete(_child);
      _child = nullptr;
    }

    if (_parent) {
      _parent->_child = nullptr;
    }

    free();

    if (DART_TEAM_NULL != _dartid &&
        DART_TEAM_ALL  != _dartid) {
      if (DART_OK != dart_team_destroy(&_dartid))
      {
        DASH_LOG_ERROR("dash::Team d'tor", "Failed to destroy DART group!");
      }
    }
  }

  inline static Team & All()
  {
    return Team::_team_all;
  }

  inline static global_unit_t GlobalUnitID()
  {
    return global_unit_t(Team::_team_all.myid());
  }

  inline static Team & Null()
  {
    return Team::_team_null;
  }

  inline static Team & Get(
    dart_team_t team_id)
  {
    if (DART_TEAM_NULL == team_id) {
      return dash::Team::Null();
    }
    if (DART_TEAM_ALL == team_id) {
      return dash::Team::All();
    }
    return *(Team::_teams[team_id]);
  }

  static void initialize()
  {
    Team::All().init_team();
  }

  static void finalize()
  {
    DASH_LOG_TRACE("Team::finalize()");

    while (Team::_teams.size() > 0) {
      Team *t = Team::_teams.begin()->second;
      delete t;
    }

    Team::All().free();
    Team::All().reset_team();
  }

  void register_deallocator(
    void * object,
    Deallocator::dealloc_function dealloc)
  {
    DASH_LOG_DEBUG_VAR("Team.register_deallocator()", object);
    _deallocs.push_back(Deallocator { object, std::move(dealloc) });
  }

  void unregister_deallocator(
    void * object,
    Deallocator::dealloc_function dealloc)
  {
    DASH_LOG_DEBUG_VAR("Team.unregister_deallocator()", object);
    _deallocs.remove(Deallocator { object, std::move(dealloc) });
  }

  void free()
  {
    DASH_LOG_DEBUG("Team.free()");
    std::list<Deallocator>::iterator next = _deallocs.begin();
    std::list<Deallocator>::iterator dealloc;
    while ((dealloc = next) != _deallocs.end()) {
      barrier();
      // List changes in iterations
      ++next;
      DASH_LOG_DEBUG_VAR("Team.free", dealloc->object);
      (dealloc->deallocator)();
    }
    _deallocs.clear();
  }

  Team & split(
    unsigned nParts);

  Team & locality_split(
    dash::util::Locality::Scope scope,
    unsigned                    num_parts);

  inline Team & locality_split(
    dart_locality_scope_t scope,
    unsigned              num_parts)
  {
    return locality_split(
             static_cast<dash::util::Locality::Scope>(
               scope),
             num_parts);
  }

  inline bool operator==(const Team & rhs) const
  {
    return _dartid == rhs._dartid;
  }

  inline bool operator!=(const Team & rhs) const
  {
    return !(operator == (rhs));
  }

  inline bool is_all() const
  {
    return operator==(All());
  }

  inline bool is_null() const
  {
    return operator==(Null());
  }

  inline bool is_leaf() const
  {
    return _child == nullptr;
  }

  inline bool is_root() const
  {
    return _parent == nullptr;
  }

  bool is_member(global_unit_t global_unit_id) const
  {
    if(!get_group()) {
      return false;
    }
    int32_t ismember;
    DASH_ASSERT_RETURNS(
      dart_group_ismember(
        _group,
        global_unit_id,
        &ismember),
      DART_OK);
    return (0 != ismember);
  }

  inline Team & parent()
  {
    if(_parent) { return *_parent; }
    else { return Null(); }
  }

  Team & sub(size_t level = 1)
  {
    Team * t = this;
    while (t && level > 0 && !(t->is_leaf())) {
      t = t->_child;
      level--;
      DASH_ASSERT_MSG(t, "node is neither leaf nor has child");
    }
    return *t;
  }

  Team & bottom()
  {
    Team *t = this;
    while (t && !(t->is_leaf())) {
      t = t->_child;
      DASH_ASSERT_MSG(t, "node is neither leaf nor has child");
    }
    return *t;
  }

  inline void barrier() const
  {
    if (!is_null()) {
      DASH_ASSERT_RETURNS(
        dart_barrier(_dartid),
        DART_OK);
    }
  }

  inline team_unit_t myid() const
  {
    return _myid;
  }

  inline size_t size() const
  {
    return _size;
  }

  inline size_t position() const
  {
    return _position;
  }

  inline size_t num_siblings() const
  {
    return _num_siblings;
  }

  inline dart_team_t dart_id() const
  {
    return _dartid;
  }

  inline size_t relative_id() const
  {
    return _position;
  }

  inline team_unit_t relative_id(
    global_unit_t global_id)
  {
    team_unit_t luid;
    DASH_ASSERT_RETURNS(
      dart_team_unit_g2l(
        _dartid,
        global_id,
        &luid),
      DART_OK);
    return luid;
  }

  inline global_unit_t global_id(
    team_unit_t local_id)
  {
    global_unit_t g_id;
    DASH_ASSERT_RETURNS(
      dart_team_unit_l2g(
        _dartid,
        local_id,
        &g_id),
      DART_OK);
    return g_id;
  }

private:

  void register_team(Team * team)
  {
    DASH_LOG_DEBUG("Team.register_team",
                   "team id:", team->_dartid);
    DASH_ASSERT_RETURNS(
      dart_team_locality_init(team->_dartid),
      DART_OK);
    dash::Team::_teams.insert(
      std::make_pair(team->_dartid, team));
  }

  void unregister_team(Team * team)
  {
    DASH_LOG_DEBUG("Team.unregister_team",
                   "team id:", team->_dartid);
//    if (team->_dartid != DART_TEAM_NULL)
    {
      DASH_ASSERT_RETURNS(
        dart_team_locality_finalize(team->_dartid),
        DART_OK);
      dash::Team::_teams.erase(
        team->_dartid);
    }
  }

  void init_team()
  {
    DASH_ASSERT_RETURNS(
      dart_team_size(_dartid, &_size),
      DART_OK);

    DASH_ASSERT_RETURNS(
      dart_team_myid(_dartid, &_myid),
      DART_OK);
  }

  void reset_team()
  {
    _myid = UNDEFINED_TEAM_UNIT_ID;
    _size = 0;
  }

private:

  dart_team_t             _dartid;
  mutable team_unit_t     _myid         = UNDEFINED_TEAM_UNIT_ID;
  mutable size_t          _size         = 0;
  Team                  * _parent       = nullptr;
  Team                  * _child        = nullptr;
  size_t                  _position     = 0;
  size_t                  _num_siblings = 0;
  mutable dart_group_t    _group        = DART_GROUP_NULL;

  std::list<Deallocator>  _deallocs;

  static std::unordered_map<dart_team_t, Team *> _teams;

  static Team _team_all;
  static Team _team_null;

}; // class Team

bool operator==(
  const Team::Deallocator & lhs,
  const Team::Deallocator & rhs);

class TeamPtr {

  std::shared_ptr<Team*> _teamptr;

}; // class TeamPtr

}  // namespace dash

namespace std {

template<>
struct iterator_traits<dash::Team::iterator> {
public:
  typedef dash::Team::iterator       value_type;
  typedef dash::Team::iterator       reference;
  typedef dash::Team::iterator       difference_type;
  typedef random_access_iterator_tag iterator_category;
};

}  // namespace std

#endif /* TEAM_H_INCLUDED */