12 #ifndef EIGEN_ASSIGN_EVALUATOR_H 13 #define EIGEN_ASSIGN_EVALUATOR_H 27 template <
typename DstEvaluator,
typename SrcEvaluator,
typename AssignFunc>
28 struct copy_using_evaluator_traits
30 typedef typename DstEvaluator::XprType Dst;
31 typedef typename Dst::Scalar DstScalar;
34 DstFlags = DstEvaluator::Flags,
35 SrcFlags = SrcEvaluator::Flags
40 DstAlignment = DstEvaluator::Alignment,
41 SrcAlignment = SrcEvaluator::Alignment,
43 JointAlignment = EIGEN_PLAIN_ENUM_MIN(DstAlignment,SrcAlignment)
48 InnerSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::SizeAtCompileTime)
49 : int(DstFlags)&
RowMajorBit ? int(Dst::ColsAtCompileTime)
50 : int(Dst::RowsAtCompileTime),
51 InnerMaxSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::MaxSizeAtCompileTime)
52 : int(DstFlags)&
RowMajorBit ? int(Dst::MaxColsAtCompileTime)
53 : int(Dst::MaxRowsAtCompileTime),
54 OuterStride = int(outer_stride_at_compile_time<Dst>::ret),
55 MaxSizeAtCompileTime = Dst::SizeAtCompileTime
59 typedef typename find_best_packet<DstScalar,Dst::SizeAtCompileTime>::type LinearPacketType;
60 typedef typename find_best_packet<DstScalar,InnerSize>::type InnerPacketType;
63 LinearPacketSize = unpacket_traits<LinearPacketType>::size,
64 InnerPacketSize = unpacket_traits<InnerPacketType>::size
69 LinearRequiredAlignment = unpacket_traits<LinearPacketType>::alignment,
70 InnerRequiredAlignment = unpacket_traits<InnerPacketType>::alignment
77 StorageOrdersAgree = (int(DstIsRowMajor) == int(SrcIsRowMajor)),
78 MightVectorize =
bool(StorageOrdersAgree)
80 &&
bool(functor_traits<AssignFunc>::PacketAccess),
81 MayInnerVectorize = MightVectorize
82 && int(InnerSize)!=
Dynamic && int(InnerSize)%int(InnerPacketSize)==0
83 && int(OuterStride)!=
Dynamic && int(OuterStride)%int(InnerPacketSize)==0
84 && (EIGEN_UNALIGNED_VECTORIZE || int(JointAlignment)>=int(InnerRequiredAlignment)),
85 MayLinearize =
bool(StorageOrdersAgree) && (int(DstFlags) & int(SrcFlags) &
LinearAccessBit),
86 MayLinearVectorize =
bool(MightVectorize) && bool(MayLinearize) && bool(DstHasDirectAccess)
87 && (EIGEN_UNALIGNED_VECTORIZE || (int(DstAlignment)>=int(LinearRequiredAlignment)) || MaxSizeAtCompileTime ==
Dynamic),
90 MaySliceVectorize = bool(MightVectorize) && bool(DstHasDirectAccess)
91 && (int(InnerMaxSize)==
Dynamic || int(InnerMaxSize)>=(EIGEN_UNALIGNED_VECTORIZE?InnerPacketSize:(3*InnerPacketSize)))
100 Traversal = int(MayLinearVectorize) && (LinearPacketSize>InnerPacketSize) ?
int(LinearVectorizedTraversal)
101 : int(MayInnerVectorize) ? int(InnerVectorizedTraversal)
102 : int(MayLinearVectorize) ? int(LinearVectorizedTraversal)
103 : int(MaySliceVectorize) ? int(SliceVectorizedTraversal)
104 : int(MayLinearize) ? int(LinearTraversal)
105 : int(DefaultTraversal),
106 Vectorized = int(Traversal) == InnerVectorizedTraversal
107 || int(Traversal) == LinearVectorizedTraversal
108 || int(Traversal) == SliceVectorizedTraversal
111 typedef typename conditional<int(Traversal)==LinearVectorizedTraversal, LinearPacketType, InnerPacketType>::type PacketType;
115 ActualPacketSize = int(Traversal)==LinearVectorizedTraversal ? LinearPacketSize
116 : Vectorized ? InnerPacketSize
118 UnrollingLimit = EIGEN_UNROLLING_LIMIT * ActualPacketSize,
119 MayUnrollCompletely = int(Dst::SizeAtCompileTime) !=
Dynamic 120 && int(Dst::SizeAtCompileTime) * (int(DstEvaluator::CoeffReadCost)+int(SrcEvaluator::CoeffReadCost)) <=
int(UnrollingLimit),
121 MayUnrollInner = int(InnerSize) !=
Dynamic 122 && int(InnerSize) * (int(DstEvaluator::CoeffReadCost)+int(SrcEvaluator::CoeffReadCost)) <=
int(UnrollingLimit)
127 Unrolling = (int(Traversal) == int(InnerVectorizedTraversal) || int(Traversal) == int(DefaultTraversal))
129 int(MayUnrollCompletely) ? int(CompleteUnrolling)
130 : int(MayUnrollInner) ? int(InnerUnrolling)
133 : int(Traversal) == int(LinearVectorizedTraversal)
134 ? ( bool(MayUnrollCompletely) && ( EIGEN_UNALIGNED_VECTORIZE || (int(DstAlignment)>=int(LinearRequiredAlignment)))
135 ? int(CompleteUnrolling)
137 : int(Traversal) == int(LinearTraversal)
138 ? ( bool(MayUnrollCompletely) ? int(CompleteUnrolling)
140 #if EIGEN_UNALIGNED_VECTORIZE
141 : int(Traversal) == int(SliceVectorizedTraversal)
142 ? ( bool(MayUnrollInner) ? int(InnerUnrolling)
148 #ifdef EIGEN_DEBUG_ASSIGN 151 std::cerr <<
"DstXpr: " <<
typeid(
typename DstEvaluator::XprType).name() << std::endl;
152 std::cerr <<
"SrcXpr: " <<
typeid(
typename SrcEvaluator::XprType).name() << std::endl;
153 std::cerr.setf(std::ios::hex, std::ios::basefield);
154 std::cerr <<
"DstFlags" <<
" = " << DstFlags <<
" (" << demangle_flags(DstFlags) <<
" )" << std::endl;
155 std::cerr <<
"SrcFlags" <<
" = " << SrcFlags <<
" (" << demangle_flags(SrcFlags) <<
" )" << std::endl;
156 std::cerr.unsetf(std::ios::hex);
157 EIGEN_DEBUG_VAR(DstAlignment)
158 EIGEN_DEBUG_VAR(SrcAlignment)
159 EIGEN_DEBUG_VAR(LinearRequiredAlignment)
160 EIGEN_DEBUG_VAR(InnerRequiredAlignment)
161 EIGEN_DEBUG_VAR(JointAlignment)
162 EIGEN_DEBUG_VAR(InnerSize)
163 EIGEN_DEBUG_VAR(InnerMaxSize)
164 EIGEN_DEBUG_VAR(LinearPacketSize)
165 EIGEN_DEBUG_VAR(InnerPacketSize)
166 EIGEN_DEBUG_VAR(ActualPacketSize)
167 EIGEN_DEBUG_VAR(StorageOrdersAgree)
168 EIGEN_DEBUG_VAR(MightVectorize)
169 EIGEN_DEBUG_VAR(MayLinearize)
170 EIGEN_DEBUG_VAR(MayInnerVectorize)
171 EIGEN_DEBUG_VAR(MayLinearVectorize)
172 EIGEN_DEBUG_VAR(MaySliceVectorize)
173 std::cerr <<
"Traversal" <<
" = " << Traversal <<
" (" << demangle_traversal(Traversal) <<
")" << std::endl;
174 EIGEN_DEBUG_VAR(SrcEvaluator::CoeffReadCost)
175 EIGEN_DEBUG_VAR(UnrollingLimit)
176 EIGEN_DEBUG_VAR(MayUnrollCompletely)
177 EIGEN_DEBUG_VAR(MayUnrollInner)
178 std::cerr <<
"Unrolling" <<
" = " << Unrolling <<
" (" << demangle_unrolling(Unrolling) <<
")" << std::endl;
179 std::cerr << std::endl;
192 template<
typename Kernel,
int Index,
int Stop>
193 struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling
196 typedef typename Kernel::DstEvaluatorType DstEvaluatorType;
197 typedef typename DstEvaluatorType::XprType DstXprType;
200 outer =
Index / DstXprType::InnerSizeAtCompileTime,
201 inner =
Index % DstXprType::InnerSizeAtCompileTime
204 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
206 kernel.assignCoeffByOuterInner(outer, inner);
207 copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, Index+1, Stop>::run(kernel);
211 template<
typename Kernel,
int Stop>
212 struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, Stop, Stop>
214 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel&) { }
217 template<
typename Kernel,
int Index_,
int Stop>
218 struct copy_using_evaluator_DefaultTraversal_InnerUnrolling
220 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel,
Index outer)
222 kernel.assignCoeffByOuterInner(outer, Index_);
223 copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, Index_+1, Stop>::run(kernel, outer);
227 template<
typename Kernel,
int Stop>
228 struct copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, Stop, Stop>
230 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel&,
Index) { }
237 template<
typename Kernel,
int Index,
int Stop>
238 struct copy_using_evaluator_LinearTraversal_CompleteUnrolling
240 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel& kernel)
242 kernel.assignCoeff(
Index);
243 copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, Index+1, Stop>::run(kernel);
247 template<
typename Kernel,
int Stop>
248 struct copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, Stop, Stop>
250 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel&) { }
257 template<
typename Kernel,
int Index,
int Stop>
258 struct copy_using_evaluator_innervec_CompleteUnrolling
261 typedef typename Kernel::DstEvaluatorType DstEvaluatorType;
262 typedef typename DstEvaluatorType::XprType DstXprType;
263 typedef typename Kernel::PacketType PacketType;
266 outer =
Index / DstXprType::InnerSizeAtCompileTime,
267 inner =
Index % DstXprType::InnerSizeAtCompileTime,
268 SrcAlignment = Kernel::AssignmentTraits::SrcAlignment,
269 DstAlignment = Kernel::AssignmentTraits::DstAlignment
272 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
274 kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, inner);
275 enum { NextIndex =
Index + unpacket_traits<PacketType>::size };
276 copy_using_evaluator_innervec_CompleteUnrolling<Kernel, NextIndex, Stop>::run(kernel);
280 template<
typename Kernel,
int Stop>
281 struct copy_using_evaluator_innervec_CompleteUnrolling<Kernel, Stop, Stop>
283 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel&) { }
286 template<
typename Kernel,
int Index_,
int Stop,
int SrcAlignment,
int DstAlignment>
287 struct copy_using_evaluator_innervec_InnerUnrolling
289 typedef typename Kernel::PacketType PacketType;
290 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel,
Index outer)
292 kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, Index_);
293 enum { NextIndex = Index_ + unpacket_traits<PacketType>::size };
294 copy_using_evaluator_innervec_InnerUnrolling<Kernel, NextIndex, Stop, SrcAlignment, DstAlignment>::run(kernel, outer);
298 template<
typename Kernel,
int Stop,
int SrcAlignment,
int DstAlignment>
299 struct copy_using_evaluator_innervec_InnerUnrolling<Kernel, Stop, Stop, SrcAlignment, DstAlignment>
301 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &,
Index) { }
310 template<
typename Kernel,
311 int Traversal = Kernel::AssignmentTraits::Traversal,
312 int Unrolling = Kernel::AssignmentTraits::Unrolling>
313 struct dense_assignment_loop;
319 template<
typename Kernel>
320 struct dense_assignment_loop<Kernel, DefaultTraversal, NoUnrolling>
322 EIGEN_DEVICE_FUNC
static void EIGEN_STRONG_INLINE run(Kernel &kernel)
324 for(
Index outer = 0; outer < kernel.outerSize(); ++outer) {
325 for(
Index inner = 0; inner < kernel.innerSize(); ++inner) {
326 kernel.assignCoeffByOuterInner(outer, inner);
332 template<
typename Kernel>
333 struct dense_assignment_loop<Kernel, DefaultTraversal, CompleteUnrolling>
335 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
337 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
338 copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
342 template<
typename Kernel>
343 struct dense_assignment_loop<Kernel, DefaultTraversal, InnerUnrolling>
345 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
347 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
349 const Index outerSize = kernel.outerSize();
350 for(
Index outer = 0; outer < outerSize; ++outer)
351 copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, 0, DstXprType::InnerSizeAtCompileTime>::run(kernel, outer);
363 template <
bool IsAligned = false>
364 struct unaligned_dense_assignment_loop
367 template <
typename Kernel>
368 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel&,
Index,
Index) {}
372 struct unaligned_dense_assignment_loop<false>
378 template <
typename Kernel>
379 static EIGEN_DONT_INLINE
void run(Kernel &kernel,
383 template <
typename Kernel>
384 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel,
389 for (
Index index = start; index < end; ++index)
390 kernel.assignCoeff(index);
394 template<
typename Kernel>
395 struct dense_assignment_loop<Kernel, LinearVectorizedTraversal, NoUnrolling>
397 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
399 const Index size = kernel.size();
400 typedef typename Kernel::Scalar Scalar;
401 typedef typename Kernel::PacketType PacketType;
403 requestedAlignment = Kernel::AssignmentTraits::LinearRequiredAlignment,
404 packetSize = unpacket_traits<PacketType>::size,
405 dstIsAligned = int(Kernel::AssignmentTraits::DstAlignment)>=int(requestedAlignment),
406 dstAlignment = packet_traits<Scalar>::AlignedOnScalar ? int(requestedAlignment)
407 : int(Kernel::AssignmentTraits::DstAlignment),
408 srcAlignment = Kernel::AssignmentTraits::JointAlignment
410 const Index alignedStart = dstIsAligned ? 0 : internal::first_aligned<requestedAlignment>(kernel.dstDataPtr(), size);
411 const Index alignedEnd = alignedStart + ((size-alignedStart)/packetSize)*packetSize;
413 unaligned_dense_assignment_loop<dstIsAligned!=0>::run(kernel, 0, alignedStart);
415 for(
Index index = alignedStart; index < alignedEnd; index += packetSize)
416 kernel.template assignPacket<dstAlignment, srcAlignment, PacketType>(index);
418 unaligned_dense_assignment_loop<>::run(kernel, alignedEnd, size);
422 template<
typename Kernel>
423 struct dense_assignment_loop<Kernel, LinearVectorizedTraversal, CompleteUnrolling>
425 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
427 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
428 typedef typename Kernel::PacketType PacketType;
430 enum { size = DstXprType::SizeAtCompileTime,
431 packetSize =unpacket_traits<PacketType>::size,
432 alignedSize = (size/packetSize)*packetSize };
434 copy_using_evaluator_innervec_CompleteUnrolling<Kernel, 0, alignedSize>::run(kernel);
435 copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, alignedSize, size>::run(kernel);
443 template<
typename Kernel>
444 struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, NoUnrolling>
446 typedef typename Kernel::PacketType PacketType;
448 SrcAlignment = Kernel::AssignmentTraits::SrcAlignment,
449 DstAlignment = Kernel::AssignmentTraits::DstAlignment
451 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
453 const Index innerSize = kernel.innerSize();
454 const Index outerSize = kernel.outerSize();
455 const Index packetSize = unpacket_traits<PacketType>::size;
456 for(
Index outer = 0; outer < outerSize; ++outer)
457 for(
Index inner = 0; inner < innerSize; inner+=packetSize)
458 kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, inner);
462 template<
typename Kernel>
463 struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, CompleteUnrolling>
465 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
467 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
468 copy_using_evaluator_innervec_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
472 template<
typename Kernel>
473 struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, InnerUnrolling>
475 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
477 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
478 typedef typename Kernel::AssignmentTraits Traits;
479 const Index outerSize = kernel.outerSize();
480 for(
Index outer = 0; outer < outerSize; ++outer)
481 copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, DstXprType::InnerSizeAtCompileTime,
482 Traits::SrcAlignment, Traits::DstAlignment>::run(kernel, outer);
490 template<
typename Kernel>
491 struct dense_assignment_loop<Kernel, LinearTraversal, NoUnrolling>
493 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
495 const Index size = kernel.size();
496 for(
Index i = 0; i < size; ++i)
497 kernel.assignCoeff(i);
501 template<
typename Kernel>
502 struct dense_assignment_loop<Kernel, LinearTraversal, CompleteUnrolling>
504 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
506 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
507 copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
515 template<
typename Kernel>
516 struct dense_assignment_loop<Kernel, SliceVectorizedTraversal, NoUnrolling>
518 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
520 typedef typename Kernel::Scalar Scalar;
521 typedef typename Kernel::PacketType PacketType;
523 packetSize = unpacket_traits<PacketType>::size,
524 requestedAlignment = int(Kernel::AssignmentTraits::InnerRequiredAlignment),
525 alignable = packet_traits<Scalar>::AlignedOnScalar || int(Kernel::AssignmentTraits::DstAlignment)>=
sizeof(Scalar),
526 dstIsAligned =
int(Kernel::AssignmentTraits::DstAlignment)>=int(requestedAlignment),
527 dstAlignment = alignable ? int(requestedAlignment)
528 : int(Kernel::AssignmentTraits::DstAlignment)
530 const Scalar *dst_ptr = kernel.dstDataPtr();
531 if((!
bool(dstIsAligned)) && (UIntPtr(dst_ptr) %
sizeof(Scalar))>0)
534 return dense_assignment_loop<Kernel,DefaultTraversal,NoUnrolling>::run(kernel);
536 const Index packetAlignedMask = packetSize - 1;
537 const Index innerSize = kernel.innerSize();
538 const Index outerSize = kernel.outerSize();
539 const Index alignedStep = alignable ? (packetSize - kernel.outerStride() % packetSize) & packetAlignedMask : 0;
540 Index alignedStart = ((!alignable) ||
bool(dstIsAligned)) ? 0 : internal::first_aligned<requestedAlignment>(dst_ptr, innerSize);
542 for(
Index outer = 0; outer < outerSize; ++outer)
544 const Index alignedEnd = alignedStart + ((innerSize-alignedStart) & ~packetAlignedMask);
546 for(
Index inner = 0; inner<alignedStart ; ++inner)
547 kernel.assignCoeffByOuterInner(outer, inner);
550 for(
Index inner = alignedStart; inner<alignedEnd; inner+=packetSize)
551 kernel.template assignPacketByOuterInner<dstAlignment, Unaligned, PacketType>(outer, inner);
554 for(
Index inner = alignedEnd; inner<innerSize ; ++inner)
555 kernel.assignCoeffByOuterInner(outer, inner);
557 alignedStart = numext::mini((alignedStart+alignedStep)%packetSize, innerSize);
562 #if EIGEN_UNALIGNED_VECTORIZE 563 template<
typename Kernel>
564 struct dense_assignment_loop<Kernel, SliceVectorizedTraversal, InnerUnrolling>
566 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
568 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
569 typedef typename Kernel::PacketType PacketType;
571 enum { size = DstXprType::InnerSizeAtCompileTime,
572 packetSize =unpacket_traits<PacketType>::size,
573 vectorizableSize = (size/packetSize)*packetSize };
575 for(
Index outer = 0; outer < kernel.outerSize(); ++outer)
577 copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, vectorizableSize, 0, 0>::run(kernel, outer);
578 copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, vectorizableSize, size>::run(kernel, outer);
595 template<
typename DstEvaluatorTypeT,
typename SrcEvaluatorTypeT,
typename Functor,
int Version = Specialized>
596 class generic_dense_assignment_kernel
599 typedef typename DstEvaluatorTypeT::XprType DstXprType;
600 typedef typename SrcEvaluatorTypeT::XprType SrcXprType;
603 typedef DstEvaluatorTypeT DstEvaluatorType;
604 typedef SrcEvaluatorTypeT SrcEvaluatorType;
605 typedef typename DstEvaluatorType::Scalar Scalar;
606 typedef copy_using_evaluator_traits<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor> AssignmentTraits;
607 typedef typename AssignmentTraits::PacketType PacketType;
610 EIGEN_DEVICE_FUNC generic_dense_assignment_kernel(DstEvaluatorType &dst,
const SrcEvaluatorType &src,
const Functor &func, DstXprType& dstExpr)
611 : m_dst(dst), m_src(src), m_functor(func), m_dstExpr(dstExpr)
613 #ifdef EIGEN_DEBUG_ASSIGN 614 AssignmentTraits::debug();
618 EIGEN_DEVICE_FUNC
Index size()
const {
return m_dstExpr.size(); }
619 EIGEN_DEVICE_FUNC
Index innerSize()
const {
return m_dstExpr.innerSize(); }
620 EIGEN_DEVICE_FUNC
Index outerSize()
const {
return m_dstExpr.outerSize(); }
621 EIGEN_DEVICE_FUNC
Index rows()
const {
return m_dstExpr.rows(); }
622 EIGEN_DEVICE_FUNC
Index cols()
const {
return m_dstExpr.cols(); }
623 EIGEN_DEVICE_FUNC
Index outerStride()
const {
return m_dstExpr.outerStride(); }
625 EIGEN_DEVICE_FUNC DstEvaluatorType& dstEvaluator() {
return m_dst; }
626 EIGEN_DEVICE_FUNC
const SrcEvaluatorType& srcEvaluator()
const {
return m_src; }
629 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignCoeff(
Index row,
Index col)
631 m_functor.assignCoeff(m_dst.coeffRef(row,col), m_src.coeff(row,col));
635 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignCoeff(
Index index)
637 m_functor.assignCoeff(m_dst.coeffRef(index), m_src.coeff(index));
641 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignCoeffByOuterInner(
Index outer,
Index inner)
643 Index row = rowIndexByOuterInner(outer, inner);
644 Index col = colIndexByOuterInner(outer, inner);
645 assignCoeff(row, col);
649 template<
int StoreMode,
int LoadMode,
typename PacketType>
650 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignPacket(
Index row,
Index col)
652 m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(row,col), m_src.template packet<LoadMode,PacketType>(row,col));
655 template<
int StoreMode,
int LoadMode,
typename PacketType>
656 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignPacket(
Index index)
658 m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(index), m_src.template packet<LoadMode,PacketType>(index));
661 template<
int StoreMode,
int LoadMode,
typename PacketType>
662 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignPacketByOuterInner(
Index outer,
Index inner)
664 Index row = rowIndexByOuterInner(outer, inner);
665 Index col = colIndexByOuterInner(outer, inner);
666 assignPacket<StoreMode,LoadMode,PacketType>(row, col);
669 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
Index rowIndexByOuterInner(
Index outer,
Index inner)
671 typedef typename DstEvaluatorType::ExpressionTraits Traits;
672 return int(Traits::RowsAtCompileTime) == 1 ? 0
673 : int(Traits::ColsAtCompileTime) == 1 ? inner
674 : int(DstEvaluatorType::Flags)&RowMajorBit ? outer
678 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
Index colIndexByOuterInner(
Index outer,
Index inner)
680 typedef typename DstEvaluatorType::ExpressionTraits Traits;
681 return int(Traits::ColsAtCompileTime) == 1 ? 0
682 : int(Traits::RowsAtCompileTime) == 1 ? inner
683 : int(DstEvaluatorType::Flags)&RowMajorBit ? inner
687 EIGEN_DEVICE_FUNC
const Scalar* dstDataPtr()
const 689 return m_dstExpr.data();
693 DstEvaluatorType& m_dst;
694 const SrcEvaluatorType& m_src;
695 const Functor &m_functor;
697 DstXprType& m_dstExpr;
704 template<
typename DstXprType,
typename SrcXprType,
typename Functor>
705 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
706 void resize_if_allowed(DstXprType &dst,
const SrcXprType& src,
const Functor &)
708 EIGEN_ONLY_USED_FOR_DEBUG(dst);
709 EIGEN_ONLY_USED_FOR_DEBUG(src);
710 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
713 template<
typename DstXprType,
typename SrcXprType,
typename T1,
typename T2>
714 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
715 void resize_if_allowed(DstXprType &dst,
const SrcXprType& src,
const internal::assign_op<T1,T2> &)
717 Index dstRows = src.rows();
718 Index dstCols = src.cols();
719 if(((dst.rows()!=dstRows) || (dst.cols()!=dstCols)))
720 dst.resize(dstRows, dstCols);
721 eigen_assert(dst.rows() == dstRows && dst.cols() == dstCols);
724 template<
typename DstXprType,
typename SrcXprType,
typename Functor>
725 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_dense_assignment_loop(DstXprType& dst,
const SrcXprType& src,
const Functor &func)
727 typedef evaluator<DstXprType> DstEvaluatorType;
728 typedef evaluator<SrcXprType> SrcEvaluatorType;
730 SrcEvaluatorType srcEvaluator(src);
734 resize_if_allowed(dst, src, func);
736 DstEvaluatorType dstEvaluator(dst);
738 typedef generic_dense_assignment_kernel<DstEvaluatorType,SrcEvaluatorType,Functor> Kernel;
739 Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
741 dense_assignment_loop<Kernel>::run(kernel);
744 template<
typename DstXprType,
typename SrcXprType>
745 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_dense_assignment_loop(DstXprType& dst,
const SrcXprType& src)
747 call_dense_assignment_loop(dst, src, internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar>());
757 template<
typename DstShape,
typename SrcShape>
struct AssignmentKind;
760 struct Dense2Dense {};
761 struct EigenBase2EigenBase {};
763 template<
typename,
typename>
struct AssignmentKind {
typedef EigenBase2EigenBase Kind; };
764 template<>
struct AssignmentKind<DenseShape,DenseShape> {
typedef Dense2Dense Kind; };
767 template<
typename DstXprType,
typename SrcXprType,
typename Functor,
768 typename Kind =
typename AssignmentKind< typename evaluator_traits<DstXprType>::Shape ,
typename evaluator_traits<SrcXprType>::Shape >::Kind,
769 typename EnableIf =
void>
778 template<
typename Dst,
typename Src>
779 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
780 void call_assignment(Dst& dst,
const Src& src)
782 call_assignment(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>());
784 template<
typename Dst,
typename Src>
785 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
786 void call_assignment(
const Dst& dst,
const Src& src)
788 call_assignment(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>());
792 template<
typename Dst,
typename Src,
typename Func>
793 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
794 void call_assignment(Dst& dst,
const Src& src,
const Func& func,
typename enable_if< evaluator_assume_aliasing<Src>::value,
void*>::type = 0)
796 typename plain_matrix_type<Src>::type tmp(src);
797 call_assignment_no_alias(dst, tmp, func);
800 template<
typename Dst,
typename Src,
typename Func>
801 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
802 void call_assignment(Dst& dst,
const Src& src,
const Func& func,
typename enable_if<!evaluator_assume_aliasing<Src>::value,
void*>::type = 0)
804 call_assignment_no_alias(dst, src, func);
809 template<
typename Dst,
template <
typename>
class StorageBase,
typename Src,
typename Func>
810 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
811 void call_assignment(NoAlias<Dst,StorageBase>& dst,
const Src& src,
const Func& func)
813 call_assignment_no_alias(dst.expression(), src, func);
817 template<
typename Dst,
typename Src,
typename Func>
818 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
819 void call_assignment_no_alias(Dst& dst,
const Src& src,
const Func& func)
822 NeedToTranspose = ( (int(Dst::RowsAtCompileTime) == 1 && int(Src::ColsAtCompileTime) == 1)
823 || (
int(Dst::ColsAtCompileTime) == 1 && int(Src::RowsAtCompileTime) == 1)
824 ) && int(Dst::SizeAtCompileTime) != 1
827 typedef typename internal::conditional<NeedToTranspose, Transpose<Dst>, Dst>::type ActualDstTypeCleaned;
828 typedef typename internal::conditional<NeedToTranspose, Transpose<Dst>, Dst&>::type ActualDstType;
829 ActualDstType actualDst(dst);
832 EIGEN_STATIC_ASSERT_LVALUE(Dst)
833 EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(ActualDstTypeCleaned,Src)
834 EIGEN_CHECK_BINARY_COMPATIBILIY(Func,
typename ActualDstTypeCleaned::Scalar,
typename Src::Scalar);
836 Assignment<ActualDstTypeCleaned,Src,Func>::run(actualDst, src, func);
838 template<
typename Dst,
typename Src>
839 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
840 void call_assignment_no_alias(Dst& dst,
const Src& src)
842 call_assignment_no_alias(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>());
845 template<
typename Dst,
typename Src,
typename Func>
846 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
847 void call_assignment_no_alias_no_transpose(Dst& dst,
const Src& src,
const Func& func)
850 EIGEN_STATIC_ASSERT_LVALUE(Dst)
851 EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst,Src)
852 EIGEN_CHECK_BINARY_COMPATIBILIY(Func,
typename Dst::Scalar,
typename Src::Scalar);
854 Assignment<Dst,Src,Func>::run(dst, src, func);
856 template<
typename Dst,
typename Src>
857 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
858 void call_assignment_no_alias_no_transpose(Dst& dst,
const Src& src)
860 call_assignment_no_alias_no_transpose(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>());
864 template<
typename Dst,
typename Src>
void check_for_aliasing(
const Dst &dst,
const Src &src);
869 template<
typename DstXprType,
typename SrcXprType,
typename Functor,
typename Weak>
870 struct Assignment<DstXprType, SrcXprType, Functor, Dense2Dense, Weak>
873 static EIGEN_STRONG_INLINE
void run(DstXprType &dst,
const SrcXprType &src,
const Functor &func)
875 #ifndef EIGEN_NO_DEBUG 876 internal::check_for_aliasing(dst, src);
879 call_dense_assignment_loop(dst, src, func);
887 template<
typename DstXprType,
typename SrcXprType,
typename Functor,
typename Weak>
888 struct Assignment<DstXprType, SrcXprType, Functor, EigenBase2EigenBase, Weak>
891 static EIGEN_STRONG_INLINE
void run(DstXprType &dst,
const SrcXprType &src,
const internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar> &)
893 Index dstRows = src.rows();
894 Index dstCols = src.cols();
895 if((dst.rows()!=dstRows) || (dst.cols()!=dstCols))
896 dst.resize(dstRows, dstCols);
898 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
904 template<
typename SrcScalarType>
906 static EIGEN_STRONG_INLINE
void run(DstXprType &dst,
const SrcXprType &src,
const internal::add_assign_op<typename DstXprType::Scalar,SrcScalarType> &)
908 Index dstRows = src.rows();
909 Index dstCols = src.cols();
910 if((dst.rows()!=dstRows) || (dst.cols()!=dstCols))
911 dst.resize(dstRows, dstCols);
913 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
917 template<
typename SrcScalarType>
919 static EIGEN_STRONG_INLINE
void run(DstXprType &dst,
const SrcXprType &src,
const internal::sub_assign_op<typename DstXprType::Scalar,SrcScalarType> &)
921 Index dstRows = src.rows();
922 Index dstCols = src.cols();
923 if((dst.rows()!=dstRows) || (dst.cols()!=dstCols))
924 dst.resize(dstRows, dstCols);
926 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
935 #endif // EIGEN_ASSIGN_EVALUATOR_H const unsigned int DirectAccessBit
Definition: Constants.h:150
Namespace containing all symbols from the Eigen library.
Definition: Core:287
const unsigned int RowMajorBit
Definition: Constants.h:61
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:33
Definition: Eigen_Colamd.h:50
const int Dynamic
Definition: Constants.h:21
const unsigned int ActualPacketAccessBit
Definition: Constants.h:100
const unsigned int LinearAccessBit
Definition: Constants.h:125