1/*
2 * ie_ghost.hpp
3 *
4 * Created on: Aug 8, 2015
5 * Author: i-bird
6 */
7
8#ifndef SRC_DECOMPOSITION_IE_GHOST_HPP_
9#define SRC_DECOMPOSITION_IE_GHOST_HPP_
10
11#include "common.hpp"
12#include "nn_processor.hpp"
13#include "Decomposition/shift_vect_converter.hpp"
14#include "Decomposition/cuda/ie_ghost_gpu.cuh"
15
16//! Processor id and box id
17struct proc_box_id
18{
19 size_t proc_id;
20 size_t box_id;
21 size_t shift_id;
22
23 //! operator to reorder
24 bool operator<(const proc_box_id & pbi) const
25 {
26 if (proc_id < pbi.proc_id)
27 {return true;}
28 else if (proc_id == pbi.proc_id)
29 {
30 return shift_id < pbi.shift_id;
31 }
32
33 return false;
34 }
35};
36
37
38/*! \brief structure that store and compute the internal and external local ghost box
39 *
40 * \tparam dim is the dimensionality of the physical domain we are going to decompose.
41 * \tparam T type of the space we decompose, Real, Integer, Complex ...
42 *
43 * \see CartDecomposition
44 *
45 */
46template<unsigned int dim, typename T, typename Memory, template<typename> class layout_base >
47class ie_ghost
48{
49 //! for each sub-domain (first vector), contain the list (nested vector) of the neighborhood processors
50 //! and for each processor contain the boxes calculated from the intersection
51 //! of the sub-domains + ghost with the near-by processor sub-domain () and the other way around
52 //! \see calculateGhostBoxes
53 openfpm::vector< openfpm::vector< Box_proc<dim,T> > > box_nn_processor_int;
54
55 //! It store the same information of box_nn_processor_int organized by processor id
56 openfpm::vector< Box_dom<dim,T> > proc_int_box;
57
58 //! External ghost boxes for this processor
59 openfpm::vector<p_box<dim,T> > vb_ext;
60
61 //! Internal ghost boxes for this processor domain
62 openfpm::vector<aggregate<unsigned int,unsigned int,unsigned int>,Memory,layout_base> vb_int;
63
64 //! Internal ghost boxes for this processor domain
65 openfpm::vector<Box<dim,T>,Memory,layout_base> vb_int_box;
66
67 //! Cell-list that store the geometrical information of the internal ghost boxes
68 CellList<dim,T,Mem_fast<Memory,int>,shift<dim,T>> geo_cell;
69
70 typedef openfpm::vector<Box<dim,T>,Memory,layout_base> proc_boxes;
71
72 //! shift vectors
73 openfpm::vector<Point<dim,T>,Memory,layout_base> shifts;
74
75 //! Temporal buffers to return temporal information for ghost_processorID
76 openfpm::vector<std::pair<size_t,size_t>> ids_p;
77
78 //! Temporal buffers to return temporal information
79 openfpm::vector<size_t> ids;
80
81 //! shift converter
82 shift_vect_converter<dim,T,Memory,layout_base> sc_convert;
83
84 //! host to device transfer
85 bool host_dev_transfer = false;
86
87 /*! \brief Given a local sub-domain i, it give the id of such sub-domain in the sent list
88 * for the processor p_id
89 *
90 * Processor 5 send its sub-domains to processor 6 and will receive the list from 6
91 *
92 * This function search if a local sub-domain has been sent to a processor p_id, if
93 * found it return at witch position is in the list of the sent sub-domains
94 *
95 * \param nn_p structure that store the processor graph as near processor
96 * \param p_id near processor rank
97 * \param i sub-domain
98 *
99 * \return Given a local sub-domain i, it give the id of such sub-domain in the sent list
100 * for the processor p_id
101 *
102 */
103 inline size_t link_ebx_ibx(const nn_prcs<dim,T,layout_base,Memory> & nn_p, size_t p_id, size_t i)
104 {
105 // Search for the correct id
106 size_t k = 0;
107 size_t p_idp = nn_p.ProctoID(p_id);
108 for (k = 0 ; k < nn_p.getSentSubdomains(p_idp).size() ; k++)
109 {
110 if (nn_p.getSentSubdomains(p_idp).get(k) == i)
111 break;
112 }
113 if (k == nn_p.getSentSubdomains(p_idp).size())
114 std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " sub-domain not found\n";
115
116 return k;
117 }
118
119 /*! \brief This is the external and internal ghost box link formula
120 *
121 * This formula is pretty important and require an extensive explanation
122 *
123 * \verbatim
124
125 +------------+
126 | |
127 | +---+---------+
128 | Processor 5| | |
129 | | E | Proc 6 |
130 | Sub 0 | 0 | |
131 | | _ | Sub 9 |
132 | | 9 | |
133 | | | |
134 | +---+---------+
135 | |
136 +------------+
137
138 * \endverbatim
139 *
140 * E0_6 is an external ghost box from the prospective of processor 5 and an internal
141 * ghost boxes from the prospective of processor 6. So for every external
142 * ghost box that processor 5 compute, exist an internal ghost box in processor 6
143 *
144 * Here we link this information with an unique id, for processor 5 and 6.
145 * Consider Processor 5 sending to processor 6
146 * its sub-domains, including the one in figure with id 0 in the list, and
147 * receive from processor 6 the sub-domain in figure as id 9. Consider also
148 * we have 16 processor. E0_9 come from the intersection of the expanded sub-domain
149 * 0 with 9 (Careful the id is related to the send and receive position in the list)
150 * and the intersection is in the sector 0
151 *
152 *
153 * The id of the external box (for processor 5) is calculated as
154 *
155 * ((k * N_b + b) * v_cl.getProcessingUnits() + p_id) * openfpm::math::pow(3,dim) + c.lin()
156 *
157 * The parameter assume a different meaning if they the formula is used for calculating
158 * external/internal ghost boxes id
159 *
160 * \param k expanded sub-domain sent/received to/from p_id ( 0 )
161 * \param b sub-domain received/sent from/to p_id ( 9 )
162 * \param p_id processor id ( 6 )
163 * \param c sector where the sub-domain b live
164 * \param N_b number of sub-domain received/sent from/to p_id
165 * \param v_cl Vcluster
166 * \param ei indicate if the formula is used to calculate external (true) or internal (false) ids
167 *
168 * \return id of the external/internal ghost
169 *
170 * \note To an explanation about the sectors see getShiftVectors
171 *
172 */
173 inline size_t ebx_ibx_form(size_t k, size_t b, size_t p_id, const comb<dim> & c ,size_t N_b, Vcluster<> & v_cl, const bool ei)
174 {
175 comb<dim> cext = c;
176
177 if (ei == true)
178 cext.sign_flip();
179
180 return ((k * N_b + b) * v_cl.getProcessingUnits() + p_id) * openfpm::math::pow(3,dim) + cext.lin();
181 }
182
183protected:
184
185 /*! \brief Here we generare the shift vectors
186 *
187 * \param domain box that describe the domain
188 *
189 */
190 void generateShiftVectors(const Box<dim,T> & domain, size_t (& bc)[dim])
191 {
192 sc_convert.generateShiftVectors(domain,bc,shifts);
193 }
194
195 /*! \brief Initialize the geo cell list structure
196 *
197 * The geo cell list structure exist to speed up the labelling the points if they fall on some
198 * internal ghost
199 *
200 * \param domain where the cell list is defined
201 * \param div number of division of the cell list
202 *
203 */
204 void Initialize_geo_cell(const Box<dim,T> & domain, const size_t (&div)[dim])
205 {
206 // Initialize the geo_cell structure
207 geo_cell.Initialize(domain,div,1);
208 }
209
210 /*! \brief Deallocate structures that identify a point to which internal ghost is located
211 *
212 */
213 void free_geo_cell()
214 {
215 geo_cell.destroy();
216 }
217
218 /*! \brief Create the box_nn_processor_int (bx part) structure
219 *
220 * For each sub-domain of the local processor it store the intersection between the enlarged
221 * sub-domain of the calling processor with the adjacent processors sub-domains (External ghost box)
222 *
223 * \param v_cl Virtual cluster
224 * \param ghost margins
225 * \param sub_domains vector of local sub-domains
226 * \param box_nn_processor it will store for each sub-domain the near processors
227 * \param nn_p contain the sub-domains of the near processors
228 *
229 * \note Are the G8_0 G9_0 G9_1 G5_0 boxes in calculateGhostBoxes
230 * \see calculateGhostBoxes
231 *
232 */
233 void create_box_nn_processor_ext(Vcluster<> & v_cl,
234 Ghost<dim,T> & ghost,
235 openfpm::vector<SpaceBox<dim,T>,Memory,layout_base> & sub_domains,
236 const openfpm::vector<openfpm::vector<long unsigned int> > & box_nn_processor,
237 const nn_prcs<dim,T,layout_base,Memory> & nn_p)
238 {
239 box_nn_processor_int.resize(sub_domains.size());
240 proc_int_box.resize(nn_p.getNNProcessors());
241
242 // For each sub-domain
243 for (size_t i = 0 ; i < sub_domains.size() ; i++)
244 {
245 SpaceBox<dim,T> sub_with_ghost = sub_domains.get(i);
246
247 // enlarge the sub-domain with the ghost
248 sub_with_ghost.enlarge(ghost);
249
250 // resize based on the number of near processors
251 box_nn_processor_int.get(i).resize(box_nn_processor.get(i).size());
252
253 // For each processor near to this sub-domain
254 for (size_t j = 0 ; j < box_nn_processor.get(i).size() ; j++)
255 {
256 // near processor
257 size_t p_id = box_nn_processor.get(i).get(j);
258
259 // used later
260 Box_dom<dim,T> & proc_int_box_g = proc_int_box.get(nn_p.ProctoID(p_id));
261
262 // Number of received sub-domains
263 size_t n_r_sub = nn_p.getNRealSubdomains(p_id);
264
265 // get the set of sub-domains, sector position, and real sub-domain id of the near processor p_id
266 const openfpm::vector< ::Box<dim,T> > & nn_processor_subdomains_g = nn_p.getNearSubdomains(p_id);
267 const openfpm::vector< comb<dim> > & nnpsg_pos = nn_p.getNearSubdomainsPos(p_id);
268 const openfpm::vector< size_t > & r_sub = nn_p.getNearSubdomainsRealId(p_id);
269
270 // used later
271 openfpm::vector< ::Box<dim,T> > & box_nn_processor_int_gg = box_nn_processor_int.get(i).get(j).bx;
272
273 // for each near processor sub-domain intersect with the enlarged local sub-domain and store it
274 for (size_t b = 0 ; b < nn_processor_subdomains_g.size() ; b++)
275 {
276 ::Box<dim,T> bi;
277 ::Box<dim,T> sub_bb(nn_processor_subdomains_g.get(b));
278
279 bool intersect = sub_with_ghost.Intersect(sub_bb,bi);
280
281 if (intersect == true)
282 {
283 struct p_box<dim,T> pb;
284
285 pb.box = bi;
286 pb.proc = p_id;
287 pb.lc_proc = nn_p.ProctoID(p_id);
288 pb.shift_id = (size_t)-1;
289
290 //
291 // Updating
292 //
293 // vb_ext
294 // box_nn_processor_int
295 // proc_int_box
296 //
297 // They all store the same information but organized in different ways
298 // read the description of each for more information
299 //
300
301 vb_ext.add(pb);
302 box_nn_processor_int_gg.add(bi);
303 proc_int_box_g.ebx.add();
304 proc_int_box_g.ebx.last().bx = bi;
305 proc_int_box_g.ebx.last().sub = i;
306 proc_int_box_g.ebx.last().cmb = nnpsg_pos.get(b);
307
308 // Search where the sub-domain i is in the sent list for processor p_id
309 size_t k = link_ebx_ibx(nn_p,p_id,i);
310
311 proc_int_box_g.ebx.last().id = ebx_ibx_form(k,r_sub.get(b),p_id,nnpsg_pos.get(b),n_r_sub,v_cl,true);
312 }
313 }
314 }
315 }
316 }
317
318
319 /*! \brief Create the box_nn_processor_int (nbx part) structure, the geo_cell list and proc_int_box
320 *
321 * This structure store for each sub-domain of this processors the boxes that come from the intersection
322 * of the near processors sub-domains enlarged by the ghost size (Internal ghost box). These boxes
323 * fill a geometrical cell list. The proc_int_box store the same information ordered by near processors
324 *
325 * \param v_cl Virtual cluster
326 * \param ghost margins
327 * \param sub_domains
328 * \param box_nn_processor sub-domains of the near processors
329 * \param nn_p structure that store the near processor sub-domains
330 *
331 * \note Are the B8_0 B9_0 B9_1 B5_0 boxes in calculateGhostBoxes
332 * \see calculateGhostBoxes
333 *
334 */
335 void create_box_nn_processor_int(Vcluster<> & v_cl,
336 Ghost<dim,T> & ghost,
337 openfpm::vector<SpaceBox<dim,T>,Memory,layout_base> & sub_domains,
338 const openfpm::vector<openfpm::vector<long unsigned int> > & box_nn_processor,
339 const nn_prcs<dim,T,layout_base,Memory> & nn_p)
340 {
341 box_nn_processor_int.resize(sub_domains.size());
342 proc_int_box.resize(nn_p.getNNProcessors());
343
344 // For each sub-domain
345 for (size_t i = 0 ; i < sub_domains.size() ; i++)
346 {
347 // For each processor contiguous to this sub-domain
348 for (size_t j = 0 ; j < box_nn_processor.get(i).size() ; j++)
349 {
350 // Near processor
351 size_t p_id = box_nn_processor.get(i).get(j);
352
353 // get the set of sub-domains of the near processor p_id
354 const openfpm::vector< ::Box<dim,T> > & nn_p_box = nn_p.getNearSubdomains(p_id);
355
356 // get the sector position for each sub-domain in the list
357 const openfpm::vector< comb<dim> > nn_p_box_pos = nn_p.getNearSubdomainsPos(p_id);
358
359 // get the real sub-domain id for each sub-domain
360 const openfpm::vector<size_t> r_sub = nn_p.getNearSubdomainsRealId(p_id);
361
362 // get the local processor id
363 size_t lc_proc = nn_p.getNearProcessor(p_id);
364
365 // For each near processor sub-domains enlarge and intersect with the local sub-domain and store the result
366 for (size_t k = 0 ; k < nn_p_box.size() ; k++)
367 {
368 // enlarge the near-processor sub-domain
369 ::Box<dim,T> n_sub = nn_p_box.get(k);
370
371 // local sub-domain
372 ::SpaceBox<dim,T> l_sub = sub_domains.get(i);
373
374 // Create a margin of ghost size around the near processor sub-domain
375 n_sub.enlarge(ghost);
376
377 // Intersect with the local sub-domain
378 Box<dim,T> b_int;
379 bool intersect = n_sub.Intersect(l_sub,b_int);
380
381 // store if it intersect
382 if (intersect == true)
383 {
384 vb_int.add();
385
386 size_t last = vb_int.size() - 1;
387
388 // the box fill with the processor id
389 vb_int.template get<proc_>(last) = p_id;
390
391 // fill the local processor id
392 vb_int.template get<lc_proc_>(last) = lc_proc;
393
394 // fill the shift id
395 vb_int.template get<shift_id_>(last) = convertShift(nn_p_box_pos.get(k));
396
397 //
398 // Updating
399 //
400 // vb_int
401 // box_nn_processor_int
402 // proc_int_box
403 //
404 // They all store the same information but organized in different ways
405 // read the description of each for more information
406 //
407
408 // add the box to the near processor sub-domain intersections
409 openfpm::vector< ::Box<dim,T> > & p_box_int = box_nn_processor_int.get(i).get(j).nbx;
410 p_box_int.add(b_int);
411 vb_int_box.add(b_int);
412
413 // store the box in proc_int_box storing from which sub-domain they come from
414 Box_sub<dim,T> sb;
415 sb.bx = b_int;
416 sb.sub = i;
417 sb.r_sub = r_sub.get(k);
418 sb.cmb = nn_p_box_pos.get(k);
419
420 size_t p_idp = nn_p.ProctoID(p_id);
421
422 // Search where the sub-domain i is in the sent list for processor p_id
423 size_t s = link_ebx_ibx(nn_p,p_id,i);
424
425 // calculate the id of the internal box
426 sb.id = ebx_ibx_form(r_sub.get(k),s,v_cl.getProcessUnitID(),nn_p_box_pos.get(k),nn_p.getSentSubdomains(p_idp).size(),v_cl,false);
427
428 Box_dom<dim,T> & pr_box_int = proc_int_box.get(nn_p.ProctoID(p_id));
429 pr_box_int.ibx.add(sb);
430
431 // update the geo_cell list
432
433 // get the cells this box span
434 const grid_key_dx<dim> p1 = geo_cell.getCellGrid_me(b_int.getP1());
435 const grid_key_dx<dim> p2 = geo_cell.getCellGrid_pe(b_int.getP2());
436
437 // Get the grid and the sub-iterator
438 auto & gi = geo_cell.getGrid();
439 grid_key_dx_iterator_sub<dim> g_sub(gi,p1,p2);
440
441 // add the box-id to the cell list
442 while (g_sub.isNext())
443 {
444 auto key = g_sub.get();
445 size_t cell = gi.LinId(key);
446
447 geo_cell.addCell(cell,vb_int.size()-1);
448
449 ++g_sub;
450 }
451 }
452 }
453 }
454 }
455
456 reorder_geo_cell();
457 }
458
459 /*! \brief in this function we reorder the list in each cells by processor id
460 *
461 * suppose in one cell we have 7 boxes each box contain the processor id
462 *
463 * 1,5,9,5,1,1,6
464 *
465 * after reorder we have the following sequence
466 *
467 * 1,1,1,5,5,6,9
468 *
469 * This simplify the procedure to get a unique list of processor ids
470 * indicating on which processor a particle must be replicated as ghost
471 *
472 */
473 void reorder_geo_cell()
474 {
475 openfpm::vector<proc_box_id> tmp_sort;
476
477 size_t div[dim];
478
479 for (size_t i = 0 ; i < dim ; i++) {div[i] = geo_cell.getDiv()[i];}
480
481 grid_sm<dim,void> gs(div);
482
483 grid_key_dx_iterator<dim> it(gs);
484
485 while (it.isNext())
486 {
487 size_t cell = gs.LinId(it.get());
488
489 size_t sz = geo_cell.getNelements(cell);
490 tmp_sort.resize(sz);
491
492 for (size_t i = 0 ; i < sz ; i++)
493 {
494 tmp_sort.get(i).box_id = geo_cell.get(cell,i);
495 tmp_sort.get(i).proc_id = vb_int.template get<proc_>(tmp_sort.get(i).box_id);
496 tmp_sort.get(i).shift_id = vb_int.template get<shift_id_>(tmp_sort.get(i).box_id);
497 }
498
499 tmp_sort.sort();
500
501 // now we set again the cell in an ordered way
502
503 for (size_t i = 0 ; i < sz ; i++)
504 {geo_cell.get(cell,i) = tmp_sort.get(i).box_id;}
505
506 ++it;
507 }
508 }
509
510public:
511
512 //! Default constructor
513 ie_ghost() {};
514
515 //! Copy constructor
516 ie_ghost(const ie_ghost<dim,T,Memory,layout_base> & ie)
517 {
518 this->operator=(ie);
519 }
520
521 //! Copy constructor
522 ie_ghost(ie_ghost<dim,T,Memory,layout_base> && ie)
523 {
524 this->operator=(ie);
525 }
526
527 //! Copy operator
528 inline ie_ghost<dim,T,Memory,layout_base> & operator=(ie_ghost<dim,T,Memory,layout_base> && ie)
529 {
530 box_nn_processor_int.swap(ie.box_nn_processor_int);
531 proc_int_box.swap(ie.proc_int_box);
532 vb_ext.swap(ie.vb_ext);
533 vb_int.swap(ie.vb_int);
534 vb_int_box.swap(ie.vb_int_box);
535 geo_cell.swap(ie.geo_cell);
536 shifts.swap(ie.shifts);
537 ids_p.swap(ie.ids_p);
538 ids.swap(ie.ids);
539
540 return *this;
541 }
542
543 //! Copy operator
544 inline ie_ghost<dim,T,Memory,layout_base> & operator=(const ie_ghost<dim,T,Memory,layout_base> & ie)
545 {
546 box_nn_processor_int = ie.box_nn_processor_int;
547 proc_int_box = ie.proc_int_box;
548 vb_ext = ie.vb_ext;
549 vb_int = ie.vb_int;
550 vb_int_box = ie.vb_int_box;
551 geo_cell = ie.geo_cell;
552 shifts = ie.shifts;
553 ids_p = ie.ids_p;
554 ids = ie.ids;
555
556 return *this;
557 }
558
559
560
561 /*! \brief duplicate this structure changing layout and Memory
562 *
563 * \return a structure with Memory type and layout changed
564 *
565 */
566 template<typename Memory2, template <typename> class layout_base2>
567 inline ie_ghost<dim,T,Memory2,layout_base2> duplicate()
568 {
569 ie_ghost<dim,T,Memory2,layout_base2> tmp;
570
571 tmp.private_get_box_nn_processor_int() = box_nn_processor_int;
572 tmp.private_get_proc_int_box() = proc_int_box;
573 tmp.private_get_vb_ext() = vb_ext;
574 tmp.private_get_vb_int() = vb_int;
575 tmp.private_get_vb_int_box() = vb_int_box;
576 tmp.private_geo_cell() = geo_cell;
577 tmp.private_get_shifts() = shifts;
578 tmp.private_get_ids_p() = ids_p;
579 tmp.private_get_ids() = ids;
580
581 return tmp;
582 }
583
584 /*! It return the shift vector
585 *
586 * Consider a domain with some ghost, at the border of the domain the
587 * ghost must be treated in a special way, depending on the periodicity
588 * of the boundary
589 *
590 \verbatim
591
592 [1,1]
593 +---------+------------------------+---------+
594 | (1,-1) | | (1,1) |
595 | | | (1,0) --> 7 | | |
596 | v | | v |
597 | 6 | | 8 |
598 +--------------------------------------------+
599 | | | |
600 | | | |
601 | | | |
602 | (-1,0) | | (1,0) |
603 | | | | | |
604 | v | (0,0) --> 4 | v |
605 | 3 | | 5 |
606 | | | |
607 B | | | A |
608 * | | | * |
609 | | | |
610 | | | |
611 | | | |
612 +--------------------------------------------+
613 | (-1,-1) | | (-1,1) |
614 | | | (-1,0) --> 1 | | |
615 | v | | v |
616 | 0 | | 2 |
617 +---------+------------------------+---------+
618
619
620 \endverbatim
621 *
622 *
623 * if a particle is bound in (1,0) linearized to 5, before communicate this particle (A in figure)
624 * must be shifted on -1.0 on x (B in figure)
625 *
626 * This function return the set of shift vectors that determine such shift, for example
627 * in the example above the shift at position 5 will be (0,-1.0)
628 *
629 * \return the shift vectors
630 *
631 */
632 const openfpm::vector<Point<dim,T>,Memory,layout_base> & getShiftVectors()
633 {
634 if (host_dev_transfer == false)
635 {
636 shifts.template hostToDevice<0>();
637 }
638
639 return shifts;
640 }
641
642 /*! It return the converted shift vector
643 *
644 * In high dimensions the number of shifts vectors explode exponentially, so we are
645 * expecting that some of the boundary is non periodic to reduce the numbers of shift
646 * vectors
647 *
648 * \return the shift vectors
649 *
650 */
651 size_t convertShift(const comb<dim> & cmb)
652 {
653 return sc_convert.linId(cmb);
654 }
655
656 /*! \brief Get the number of Internal ghost boxes for one processor
657 *
658 * \param id near processor list id (the id go from 0 to getNNProcessor())
659 * \return the number of internal ghost
660 *
661 */
662 inline size_t getProcessorNIGhost(size_t id) const
663 {
664 return proc_int_box.get(id).ibx.size();
665 }
666
667 /*! \brief Get the number of External ghost boxes for one processor id
668 *
669 * \param id near processor list id (the id go from 0 to getNNProcessor())
670 * \return the number of external ghost
671 *
672 */
673 inline size_t getProcessorNEGhost(size_t id) const
674 {
675 return proc_int_box.get(id).ebx.size();
676 }
677
678 /*! \brief Get the j Internal ghost box for one processor
679 *
680 * \param id near processor list id (the id go from 0 to getNNProcessor())
681 * \param j box (each near processor can produce more than one internal ghost box)
682 * \return the box
683 *
684 */
685 inline const ::Box<dim,T> & getProcessorIGhostBox(size_t id, size_t j) const
686 {
687 return proc_int_box.get(id).ibx.get(j).bx;
688 }
689
690 /*! \brief Get the j External ghost box
691 *
692 * \param id near processor list id (the id go from 0 to getNNProcessor())
693 * \param j box (each near processor can produce more than one external ghost box)
694 * \return the box
695 *
696 */
697 inline const ::Box<dim,T> & getProcessorEGhostBox(size_t id, size_t j) const
698 {
699 return proc_int_box.get(id).ebx.get(j).bx;
700 }
701
702 /*! \brief Get the j External ghost box sector
703 *
704 * \param id near processor list id (the id go from 0 to getNNProcessor())
705 * \param j box (each near processor can produce more than one external ghost box)
706 * \return the sector
707 *
708 */
709 inline const comb<dim> & getProcessorEGhostPos(size_t id, size_t j) const
710 {
711 return proc_int_box.get(id).ebx.get(j).cmb;
712 }
713
714 /*! \brief Get the ghost box sector of the external ghost box linked with the j internal ghost box
715 *
716 * \param id near processor list id (the id go from 0 to getNNProcessor())
717 * \param j box (each near processor can produce more than one internal ghost box)
718 * \return the sector
719 *
720 */
721 inline const comb<dim> & getProcessorIGhostPos(size_t id, size_t j) const
722 {
723 return proc_int_box.get(id).ibx.get(j).cmb;
724 }
725
726 /*! \brief Get the j Internal ghost box id
727 *
728 * Every internal ghost box has a linked external ghost box, because they overlap
729 * and they must contain the same information (Think on a ghost_get). So if exist
730 * an internal ghost box with id x, exist also an external ghost box with id x
731 *
732 * \param id near processor list id (the id go from 0 to getNNProcessor())
733 * \param j box (each near processor can produce more than one internal ghost box)
734 * \return the box id
735 *
736 */
737 inline size_t getProcessorIGhostId(size_t id, size_t j) const
738 {
739 return proc_int_box.get(id).ibx.get(j).id;
740 }
741
742 /*! \brief Get the j External ghost box id
743 *
744 * Every external ghost box has a linked internal ghost box, because they overlap
745 * and they must contain the same information (Think on a ghost_get). So if exist
746 * an internal ghost box with id x, exist also an external ghost box with id x
747 *
748 * \param id near processor list id (the id go from 0 to getNNProcessor())
749 * \param j box (each near processor can produce more than one external ghost box)
750 * \return the box
751 *
752 */
753 inline size_t getProcessorEGhostId(size_t id, size_t j) const
754 {
755 return proc_int_box.get(id).ebx.get(j).id;
756 }
757
758 /*! \brief Get the sub-domain send-id at witch belong the internal ghost box
759 *
760 * The internal ghost box is create from the intersection a local sub-domain
761 * and an extended sub-domain communicated from another processor. This function
762 * return the id of the sub-domain in the receiving list
763 *
764 * \param id adjacent processor list id (the id go from 0 to getNNProcessor())
765 * \param j box (each near processor can produce more than one internal ghost box)
766 * \return sub-domain at which belong the internal ghost box
767 *
768 */
769 inline size_t getProcessorIGhostSSub(size_t id, size_t j) const
770 {
771 return proc_int_box.get(id).ibx.get(j).r_sub;
772 }
773
774 /*! \brief Get the local sub-domain at witch belong the internal ghost box
775 *
776 * \param id adjacent processor list id (the id go from 0 to getNNProcessor())
777 * \param j box (each near processor can produce more than one internal ghost box)
778 * \return sub-domain at which belong the internal ghost box
779 *
780 */
781 inline size_t getProcessorIGhostSub(size_t id, size_t j) const
782 {
783 return proc_int_box.get(id).ibx.get(j).sub;
784 }
785
786 /*! \brief Get the local sub-domain at witch belong the external ghost box
787 *
788 * \param id near processor list id (the id go from 0 to getNNProcessor())
789 * \param j box (each near processor can produce more than one external ghost box)
790 * \return sub-domain at which belong the external ghost box
791 *
792 */
793 inline size_t getProcessorEGhostSub(size_t id, size_t j) const
794 {
795 return proc_int_box.get(id).ebx.get(j).sub;
796 }
797
798 /*! \brief Return the total number of the calculated internal ghost boxes
799 *
800 * \return the number of internal ghost boxes
801 *
802 */
803 inline size_t getNIGhostBox() const
804 {
805 return vb_int.size();
806 }
807
808 /*! \brief Given the internal ghost box id, it return the internal ghost box
809 *
810 * \param b_id internal ghost box id
811 *
812 * \return the internal ghost box
813 *
814 */
815 inline ::Box<dim,T> getIGhostBox(size_t b_id) const
816 {
817 return vb_int_box.get(b_id);
818 }
819
820 /*! \brief Given the internal ghost box id, it return the near processor at witch belong
821 * or the near processor that produced this internal ghost box
822 *
823 * \param b_id internal ghost box id
824 *
825 * \return the processor id of the ghost box
826 *
827 */
828 inline size_t getIGhostBoxProcessor(size_t b_id) const
829 {
830 return vb_int.template get<proc_>(b_id);
831 }
832
833 /*! \brief Get the number of the calculated external ghost boxes
834 *
835 * \return the number of external ghost boxes
836 *
837 */
838 inline size_t getNEGhostBox() const
839 {
840 return vb_ext.size();
841 }
842
843 /*! \brief Given the external ghost box id, it return the external ghost box
844 *
845 * \param b_id external ghost box id
846 *
847 * \return the external ghost box
848 *
849 */
850 inline ::Box<dim,T> getEGhostBox(size_t b_id) const
851 {
852 return vb_ext.get(b_id).box;
853 }
854
855 /*! \brief Given the external ghost box id, it return the near processor at witch belong
856 * or the near processor that produced this external ghost box
857 *
858 * \param b_id external ghost box id
859 *
860 * \return the processor id of the external ghost box
861 *
862 */
863 inline size_t getEGhostBoxProcessor(size_t b_id) const
864 {
865 return vb_ext.get(b_id).proc;
866 }
867
868 /*! /brief Given a point it return the set of boxes in which the point fall
869 *
870 * \param p Point to check
871 *
872 * \return An iterator with the id's of the internal boxes in which the point fall
873 *
874 */
875 auto getInternalIDBoxes(Point<dim,T> & p) -> decltype(geo_cell.getCellIterator(geo_cell.getCell(p)))
876 {
877 return geo_cell.getCellIterator(geo_cell.getCell(p));
878 }
879
880 /*! \brief if the point fall into the ghost of some near processor it return the processors id's in which
881 * it fall
882 *
883 * \param p Point
884 * \return iterator of the processors id's
885 *
886 */
887 inline auto labelPoint(Point<dim,T> & p) -> decltype(geo_cell.getCellIterator(geo_cell.getCell(p)))
888 {
889 return geo_cell.getCellIterator(geo_cell.getCell(p));
890 }
891
892 /*! \brief Get the number of processor a particle must sent
893 *
894 * \param p position of the particle
895 *
896 */
897 template<typename output_type> inline void ghost_processor_ID(const Point<dim,T> & p, output_type & output, unsigned int base, unsigned int pi)
898 {
899 ID_operation<output_type> op(output);
900
901 ghost_processorID_general_impl(p,base,pi,geo_cell,vb_int_box,vb_int,op);
902 }
903
904 /*! \brief Get the number of processor a particle must sent
905 *
906 * \param p position of the particle
907 *
908 */
909 inline unsigned int ghost_processorID_N(const Point<dim,T> & p)
910 {
911 return ghost_processorID_N_impl(p,geo_cell,vb_int_box,vb_int);
912 }
913
914 /*! \brief Given a position it return if the position belong to any neighborhood processor ghost
915 * (Internal ghost)
916 *
917 * if the particle come from an internal ghost from the periodicity of the domain, position must be shifted
918 * this function return the id of the shift vector
919 *
920 * \see getShiftVector
921 *
922 * \tparam id type of id to get box_id processor_id lc_processor_id shift_id
923 *
924 * \param p Particle position
925 * \param opt intersection boxes of the same processor can overlap, so in general the function
926 * can produce more entry with the same processor, the UNIQUE option eliminate double entries
927 * (UNIQUE) is for particle data (MULTIPLE) is for grid data [default MULTIPLE]
928 * \return return the processor ids (not the rank, the id in the near processor list)
929 *
930 */
931 template <typename id1, typename id2> inline const openfpm::vector<std::pair<size_t,size_t>> ghost_processorID_pair(Point<dim,T> & p, const int opt = MULTIPLE)
932 {
933 ids_p.clear();
934
935 // Check with geo-cell if a particle is inside one Cell containing boxes
936
937 auto cell_it = geo_cell.getCellIterator(geo_cell.getCell(p));
938
939 // For each element in the cell, check if the point is inside the box
940 // if it is, store the processor id
941 while (cell_it.isNext())
942 {
943 size_t bid = cell_it.get();
944
945 if (Box<dim,T>(vb_int_box.get(bid)).isInsideNP(p) == true)
946 {
947 ids_p.add(std::pair<size_t,size_t>(id1::id(vb_int.get(bid),bid),id2::id(vb_int.get(bid),bid)));
948 }
949
950 ++cell_it;
951 }
952
953 // Make the id unique
954 if (opt == UNIQUE)
955 {
956 ids_p.sort();
957 ids_p.unique();
958 }
959
960 return ids_p;
961 }
962
963 /*! \brief Given a position it return if the position belong to any neighborhood processor ghost
964 * (Internal ghost)
965 *
966 * if the particle come from an internal ghost from the periodicity of the domain, position must be shifted
967 * this function return the id of the shift vector
968 *
969 * \see getShiftVector
970 *
971 * \tparam id type of id to get box_id processor_id lc_processor_id shift_id
972 * \param p Particle position
973 * \param opt intersection boxes of the same processor can overlap, so in general the function
974 * can produce more entry with the same processor, the UNIQUE option eliminate double entries
975 * (UNIQUE) is for particle data (MULTIPLE) is for grid data [default MULTIPLE]
976 *
977 * \return the processor ids
978 *
979 */
980 template <typename id> inline const openfpm::vector<size_t> ghost_processorID(const Point<dim,T> & p, const int opt = MULTIPLE)
981 {
982 ids.clear();
983
984 // Check with geo-cell if a particle is inside one Cell containing boxes
985
986 auto cell_it = geo_cell.getCellIterator(geo_cell.getCell(p));
987
988 // For each element in the cell, check if the point is inside the box
989 // if it is, store the processor id
990 while (cell_it.isNext())
991 {
992 size_t bid = cell_it.get();
993
994 if (Box<dim,T>(vb_int_box.get(bid)).isInsideNP(p) == true)
995 {
996 ids.add(id::id(vb_int.get(bid),bid));
997 }
998
999 ++cell_it;
1000 }
1001
1002 // Make the id unique
1003 if (opt == UNIQUE)
1004 {
1005 ids_p.sort();
1006 ids_p.unique();
1007 }
1008
1009 return ids;
1010 }
1011
1012 /*! \brief Given a position it return if the position belong to any neighborhood processor ghost
1013 * (Internal ghost)
1014 *
1015 * \tparam id1 first index type to get box_id processor_id lc_processor_id
1016 * \tparam id2 second index type to get box_id processor_id lc_processor_id
1017 *
1018 * \param p Particle position
1019 * \param opt indicate if the entries in the vector must be unique
1020 *
1021 * \return a vector of pair containing the requested information
1022 *
1023 */
1024 template<typename id1, typename id2, typename Mem> inline const openfpm::vector<std::pair<size_t,size_t>> & ghost_processorID_pair(const encapc<1,Point<dim,T>,Mem> & p, const int opt = MULTIPLE)
1025 {
1026 ids_p.clear();
1027
1028 // Check with geo-cell if a particle is inside one Cell containing boxes
1029
1030 auto cell_it = geo_cell.getCellIterator(geo_cell.getCell(p));
1031
1032 // For each element in the cell, check if the point is inside the box
1033 // if it is, store the processor id
1034 while (cell_it.isNext())
1035 {
1036 size_t bid = cell_it.get();
1037
1038 if (Box<dim,T>(vb_int_box.get(bid)).isInsideNP(p) == true)
1039 {
1040 ids_p.add(std::pair<size_t,size_t>(id1::id(vb_int.get(bid),bid),id2::id(vb_int.get(bid),bid)));
1041 }
1042
1043 ++cell_it;
1044 }
1045
1046 // Make the id unique
1047 if (opt == UNIQUE)
1048 {
1049 ids_p.sort();
1050 ids_p.unique();
1051 }
1052
1053 return ids_p;
1054 }
1055
1056 /*! \brief Given a position it return if the position belong to any neighborhood processor ghost
1057 * (Internal ghost)
1058 *
1059 * \tparam id type of if to get box_id processor_id lc_processor_id
1060 *
1061 * \param p Particle position
1062 * \param opt it indicate if the entry in the vector must be unique or not
1063 *
1064 * \return the processor ids
1065 *
1066 */
1067 template<typename id, typename Mem> inline const openfpm::vector<size_t> & ghost_processorID(const encapc<1,Point<dim,T>,Mem> & p, const int opt = MULTIPLE)
1068 {
1069 ids.clear();
1070
1071 // Check with geo-cell if a particle is inside one Cell containing boxes
1072
1073 auto cell_it = geo_cell.getCellIterator(geo_cell.getCell(p));
1074
1075 // For each element in the cell, check if the point is inside the box
1076 // if it is, store the processor id
1077 while (cell_it.isNext())
1078 {
1079 size_t bid = cell_it.get();
1080
1081 if (vb_int.get(bid).box.isInsideNP(p) == true)
1082 {
1083 ids.add(id::id(vb_int.get(bid),bid));
1084 }
1085
1086 ++cell_it;
1087 }
1088
1089 // Make the id unique
1090 if (opt == UNIQUE)
1091 {
1092 ids_p.sort();
1093 ids_p.unique();
1094 }
1095
1096 return ids;
1097 }
1098
1099 /*! \brief write the information about the ghost in vtk format
1100 *
1101 * 1) internal_ghost_X.vtk Internal ghost boxes for the local processor (X)
1102 * 2) external_ghost_X.vtk External ghost boxes for the local processor (X)
1103 *
1104 * \param output directory
1105 * \param p_id processor rank
1106 *
1107 *
1108 * \return true if the write succeed
1109 *
1110 */
1111 bool write(std::string output, size_t p_id) const
1112 {
1113 //! internal_ghost_X.vtk Internal ghost boxes for the local processor (X)
1114 VTKWriter<openfpm::vector<::Box<dim,T>>,VECTOR_BOX> vtk_box3;
1115 for (size_t p = 0 ; p < box_nn_processor_int.size() ; p++)
1116 {
1117 for (size_t s = 0 ; s < box_nn_processor_int.get(p).size() ; s++)
1118 {
1119 vtk_box3.add(box_nn_processor_int.get(p).get(s).nbx);
1120 }
1121 }
1122 vtk_box3.write(output + std::string("internal_ghost_") + std::to_string(p_id) + std::string(".vtk"));
1123
1124 //! external_ghost_X.vtk External ghost boxes for the local processor (X)
1125 VTKWriter<openfpm::vector<::Box<dim,T>>,VECTOR_BOX> vtk_box4;
1126 for (size_t p = 0 ; p < box_nn_processor_int.size() ; p++)
1127 {
1128 for (size_t s = 0 ; s < box_nn_processor_int.get(p).size() ; s++)
1129 {
1130 vtk_box4.add(box_nn_processor_int.get(p).get(s).bx);
1131 }
1132 }
1133 vtk_box4.write(output + std::string("external_ghost_") + std::to_string(p_id) + std::string(".vtk"));
1134
1135 return true;
1136 }
1137
1138 /*! \brief Check if the ie_ghosts contain the same information
1139 *
1140 * \param ig Element to check
1141 *
1142 * \return true if they are equal
1143 *
1144 */
1145 bool is_equal(ie_ghost<dim,T,Memory,layout_base> & ig)
1146 {
1147 if (getNEGhostBox() != ig.getNEGhostBox())
1148 return false;
1149
1150 if (getNIGhostBox() != ig.getNIGhostBox())
1151 return false;
1152
1153 for (size_t i = 0 ; i < getNIGhostBox() ; i++)
1154 {
1155 if (getIGhostBox(i) != ig.getIGhostBox(i))
1156 return false;
1157 if (getIGhostBoxProcessor(i) != ig.getIGhostBoxProcessor(i))
1158 return false;
1159 }
1160
1161 for (size_t i = 0 ; i < proc_int_box.size() ; i++)
1162 {
1163 if (getProcessorNIGhost(i) != ig.getProcessorNIGhost(i))
1164 return false;
1165 for (size_t j = 0 ; j < getProcessorNIGhost(i) ; j++)
1166 {
1167 if (getProcessorIGhostBox(i,j) != ig.getProcessorIGhostBox(i,j))
1168 return false;
1169 if (getProcessorIGhostId(i,j) != ig.getProcessorIGhostId(i,j))
1170 return false;
1171 if (getProcessorIGhostSub(i,j) != ig.getProcessorIGhostSub(i,j))
1172 return false;
1173 }
1174 }
1175
1176 for (size_t i = 0 ; i < getNEGhostBox() ; i++)
1177 {
1178 if (getEGhostBox(i) != ig.getEGhostBox(i))
1179 return false;
1180 if (getEGhostBoxProcessor(i) != ig.getEGhostBoxProcessor(i))
1181 return false;
1182 }
1183
1184 for (size_t i = 0 ; i < proc_int_box.size() ; i++)
1185 {
1186 if (getProcessorNEGhost(i) != ig.getProcessorNEGhost(i))
1187 return false;
1188 for (size_t j = 0 ; j < getProcessorNEGhost(i) ; j++)
1189 {
1190 if (getProcessorEGhostBox(i,j) != ig.getProcessorEGhostBox(i,j))
1191 return false;
1192 if (getProcessorEGhostId(i,j) != ig.getProcessorEGhostId(i,j))
1193 return false;
1194 if (getProcessorEGhostSub(i,j) != ig.getProcessorEGhostSub(i,j))
1195 return false;
1196 }
1197 }
1198
1199 return true;
1200 }
1201
1202 /*! \brief Check if the ie_loc_ghosts contain the same information with the exception of the ghost part
1203 * It is anyway required that the ghost come from the same sub-domains decomposition
1204 *
1205 * \param ig Element to check
1206 *
1207 * \return true if they are equal
1208 *
1209 */
1210 bool is_equal_ng(ie_ghost<dim,T,Memory,layout_base> & ig)
1211 {
1212 return true;
1213 }
1214
1215 /*! \brief Reset the nn_prcs structure
1216 *
1217 */
1218 void reset()
1219 {
1220 box_nn_processor_int.clear();
1221 proc_int_box.clear();
1222 vb_ext.clear();
1223 vb_int.clear();
1224 vb_int_box.clear();
1225 geo_cell.clear();
1226 shifts.clear();
1227 ids_p.clear();
1228 ids.clear();
1229 }
1230
1231 /*! \brief Return the internal data structure box_nn_processor_int
1232 *
1233 * \return box_nn_processor_int
1234 *
1235 */
1236 inline openfpm::vector< openfpm::vector< Box_proc<dim,T> > > & private_get_box_nn_processor_int()
1237 {
1238 return box_nn_processor_int;
1239 }
1240
1241 /*! \brief Return the internal data structure proc_int_box
1242 *
1243 * \return proc_int_box
1244 *
1245 */
1246 inline openfpm::vector< Box_dom<dim,T> > & private_get_proc_int_box()
1247 {
1248 return proc_int_box;
1249 }
1250
1251 /*! \brief Return the internal data structure vb_ext
1252 *
1253 * \return vb_ext
1254 *
1255 */
1256 inline openfpm::vector<p_box<dim,T> > & private_get_vb_ext()
1257 {
1258 return vb_ext;
1259 }
1260
1261 /*! \brief Return the internal data structure vb_int
1262 *
1263 * \return vb_int
1264 *
1265 */
1266 inline openfpm::vector<aggregate<unsigned int,unsigned int,unsigned int>,Memory,layout_base> &
1267 private_get_vb_int()
1268 {
1269 return vb_int;
1270 }
1271
1272 /*! \brief Return the internal data structure vb_int_box
1273 *
1274 * \return vb_int_box
1275 *
1276 */
1277 inline openfpm::vector<Box<dim,T>,Memory,layout_base> &
1278 private_get_vb_int_box()
1279 {
1280 return vb_int_box;
1281 }
1282
1283 /*! \brief Return the internal data structure proc_int_box
1284 *
1285 * \return proc_int_box
1286 *
1287 */
1288 inline CellList<dim,T,Mem_fast<Memory,int>,shift<dim,T>> &
1289 private_geo_cell()
1290 {
1291 return geo_cell;
1292 }
1293
1294 /*! \brief Return the internal data structure shifts
1295 *
1296 * \return shifts
1297 *
1298 */
1299 inline openfpm::vector<Point<dim,T>,Memory,layout_base> &
1300 private_get_shifts()
1301 {
1302 return shifts;
1303 }
1304
1305 /*! \brief Return the internal data structure ids_p
1306 *
1307 * \return ids_p
1308 *
1309 */
1310 inline openfpm::vector<std::pair<size_t,size_t>> &
1311 private_get_ids_p()
1312 {
1313 return ids_p;
1314 }
1315
1316 /*! \brief Return the internal data structure ids_p
1317 *
1318 * \return ids_p
1319 *
1320 */
1321 inline openfpm::vector<size_t> &
1322 private_get_ids()
1323 {
1324 return ids;
1325 }
1326
1327 /*! \brief toKernel() Convert this data-structure into a kernel usable data-structure
1328 *
1329 * \return
1330 *
1331 */
1332 ie_ghost_gpu<dim,T,Memory,layout_base> toKernel()
1333 {
1334 if (host_dev_transfer == false)
1335 {
1336 geo_cell.hostToDevice();
1337 vb_int_box.template hostToDevice<0,1>();
1338 vb_int.template hostToDevice<0,1,2>();
1339 shifts.template hostToDevice<0>();
1340
1341 host_dev_transfer = true;
1342 }
1343
1344 ie_ghost_gpu<dim,T,Memory,layout_base> igg(geo_cell.toKernel(),
1345 vb_int_box.toKernel(),
1346 vb_int.toKernel());
1347
1348 return igg;
1349 }
1350
1351 /*! \brief Notify that the next toKernel() data-structures must be re-offloaded
1352 *
1353 *
1354 */
1355 void reset_host_dev_transfer()
1356 {
1357 host_dev_transfer = false;
1358 }
1359};
1360
1361
1362#endif /* SRC_DECOMPOSITION_IE_GHOST_HPP_ */
1363