[casacore] r21577 committed - Optimized IN operator (for CASA)

[casa...@googlecode.com]

Revision: 21577
Author: gervandiepen
Date: Wed Mar 18 15:00:56 2015 UTC
Log: Optimized IN operator (for CASA)

https://code.google.com/p/casacore/source/detail?r=21577

Modified:
/branches/nov14/tables/TaQL/ExprLogicNode.cc
/branches/nov14/tables/TaQL/ExprLogicNode.h
/branches/nov14/tables/TaQL/ExprNode.cc
/branches/nov14/tables/TaQL/ExprNode.h
/branches/nov14/tables/TaQL/ExprNodeArray.cc
/branches/nov14/tables/TaQL/ExprNodeArray.h
/branches/nov14/tables/TaQL/TaQLNodeHandler.cc
/branches/nov14/tables/TaQL/UDFBase.h

=======================================
--- /branches/nov14/tables/TaQL/ExprLogicNode.cc Wed Dec 10 08:06:42 2014
UTC
+++ /branches/nov14/tables/TaQL/ExprLogicNode.cc Wed Mar 18 15:00:56 2015
UTC
@@ -282,14 +282,44 @@
}


-TableExprNodeINInt::TableExprNodeINInt (const TableExprNodeRep& node)
-: TableExprNodeBinary (NTBool, node, OtIN)
+TableExprNodeINInt::TableExprNodeINInt (const TableExprNodeRep& node,
+ Bool doTracing)
+: TableExprNodeBinary (NTBool, node, OtIN),
+ itsDoTracing (doTracing)
{}
+void TableExprNodeINInt::convertConstChild()
+{
+ if (rnode_p->isConstant() && rnode_p->valueType() == VTArray) {
+ // Convert set/array to a Bool array (for direct lookup) if the range
+ // is sufficiently small.
+ Array<Int64> arr = rnode_p->getArrayInt(0);
+ minMax (itsMin, itsMax, arr);
+ Int64 sz = itsMax - itsMin + 1;
+ if (sz <= 1024*1024) {
+ itsIndex.resize (sz);
+ itsIndex = False;
+ Array<Int64>::const_iterator arrend = arr.end();
+ for (Array<Int64>::const_iterator iter=arr.begin();
+ iter!=arrend; ++iter) {
+ itsIndex[*iter - itsMin] = True;
+ }
+ if (itsDoTracing) {
+ cout << " created IN index of size " << sz
+ <<" offset=" << itsMin << endl;
+ }
+ }
+ }
+}
TableExprNodeINInt::~TableExprNodeINInt()
{}
Bool TableExprNodeINInt::getBool (const TableExprId& id)
{
- return rnode_p->hasInt (id, lnode_p->getInt (id));
+ Int64 val = lnode_p->getInt (id);
+ if (itsIndex.size() > 0) {
+ if (val < itsMin || val > itsMax) return False;
+ return itsIndex[val - itsMin];
+ }
+ return rnode_p->hasInt (id, val);
}

TableExprNodeINDouble::TableExprNodeINDouble (const TableExprNodeRep& node)
=======================================
--- /branches/nov14/tables/TaQL/ExprLogicNode.h Wed Dec 10 08:06:42 2014 UTC
+++ /branches/nov14/tables/TaQL/ExprLogicNode.h Wed Mar 18 15:00:56 2015 UTC
@@ -797,9 +797,17 @@
class TableExprNodeINInt : public TableExprNodeBinary
{
public:
- TableExprNodeINInt (const TableExprNodeRep&);
- ~TableExprNodeINInt();
- Bool getBool (const TableExprId& id);
+ TableExprNodeINInt (const TableExprNodeRep&, Bool doTracing=False);
+ virtual ~TableExprNodeINInt();
+ virtual void convertConstChild();
+ virtual Bool getBool (const TableExprId& id);
+private:
+ Bool itsDoTracing;
+ //# If the right node is constant and its range is sufficiently small,
+ //# it is turned into a Bool index for linear lookup time.
+ Block<Bool> itsIndex;
+ Int64 itsMin;
+ Int64 itsMax;
};


=======================================
--- /branches/nov14/tables/TaQL/ExprNode.cc Wed Dec 10 08:06:42 2014 UTC
+++ /branches/nov14/tables/TaQL/ExprNode.cc Wed Mar 18 15:00:56 2015 UTC
@@ -241,7 +241,8 @@
return left.newOR (right.node_p);
}

-TableExprNode TableExprNode::in (const TableExprNodeSet& set) const
+TableExprNode TableExprNode::in (const TableExprNodeSet& set,
+ const TaQLStyle& style) const
{
// An empty set never matches.
// Note it makes it possible to use an empty set that has
@@ -251,7 +252,7 @@
}
set.checkEqualDataTypes();
TableExprNodeSet setcp = set;
- return newIN (setcp.setOrArray());
+ return newIN (setcp.setOrArray(), style);
}

TableExprNode TableExprNode::useUnit (const Unit& unit) const
@@ -814,14 +815,33 @@
}


-TableExprNodeRep* TableExprNode::newIN (TableExprNodeRep* right) const
+TableExprNodeRep* TableExprNode::newIN (TableExprNodeRep* right,
+ const TaQLStyle& style) const
{
+ // Use EQ if a single value is used (scalar or single set element).
TableExprNodeRep::ValueType vtRight = right->valueType();
- if (vtRight != TableExprNodeRep::VTArray
- && vtRight != TableExprNodeRep::VTSet
- && vtRight != TableExprNodeRep::VTScalar) {
- throw (TableInvExpr
- ("Right operand of IN has to be a scalar, array or
set"));
+ if (vtRight == TableExprNodeRep::VTScalar) {
+ return newEQ (right);
+ } else if (vtRight == TableExprNodeRep::VTArray) {
+ TableExprNodeSet* set = dynamic_cast<TableExprNodeSet*>(right);
+ if (set) {
+ if (set->isSingle() && set->nelements() == 1 &&
+ ! set->hasArrays()) {
+ TableExprNodeRep* snode = (*set)[0].start();
+ return newEQ (snode);
+ }
+ } else {
+ TableExprNodeArray* arr = dynamic_cast<TableExprNodeArray*>(right);
+ if (arr) {
+ TableExprNodeRep* sca = arr->makeConstantScalar();
+ if (sca) {
+ return newEQ (sca);
+ }
+ }
+ }
+ } else if (vtRight != TableExprNodeRep::VTSet) {
+ throw (TableInvExpr
+ ("Right operand of IN has to be a scalar, array or set"));
}
TableExprNodeRep::NodeDataType dtype = node_p->dataType();
TableExprNodeRep::NodeDataType rdtype = right->dataType();
@@ -848,7 +868,7 @@
if (node.valueType() == TableExprNodeRep::VTScalar) {
switch (node.dataType()) {
case TableExprNodeRep::NTInt:
- tsnptr = new TableExprNodeINInt (node);
+ tsnptr = new TableExprNodeINInt (node, style.doTracing());
break;
case TableExprNodeRep::NTDouble:
tsnptr = new TableExprNodeINDouble (node);
@@ -886,7 +906,7 @@
throwInvDT("in array IN-operator");
}
}
- return TableExprNodeBinary::fillNode (tsnptr, node_p, right, False);
+ return TableExprNodeBinary::fillNode (tsnptr, node_p, right, True);
}

TableExprNodeRep* TableExprNode::newOR (TableExprNodeRep* right) const
=======================================
--- /branches/nov14/tables/TaQL/ExprNode.h Wed Dec 10 08:06:42 2014 UTC
+++ /branches/nov14/tables/TaQL/ExprNode.h Wed Mar 18 15:00:56 2015 UTC
@@ -827,8 +827,10 @@
// The IN operator to test if a value is contained in an array or set.
// The array can also be a scalar.
// <group>
- TableExprNode in (const TableExprNode& array) const;
- TableExprNode in (const TableExprNodeSet& set) const;
+ TableExprNode in (const TableExprNode& array,
+ const TaQLStyle& = TaQLStyle(0)) const;
+ TableExprNode in (const TableExprNodeSet& set,
+ const TaQLStyle& = TaQLStyle(0)) const;
// </group>

// Use a unit for the given TableExprNode.
@@ -1110,7 +1112,8 @@
TableExprNodeRep* newNE (TableExprNodeRep* right) const;
TableExprNodeRep* newGE (TableExprNodeRep* right) const;
TableExprNodeRep* newGT (TableExprNodeRep* right) const;
- TableExprNodeRep* newIN (TableExprNodeRep* right) const;
+ TableExprNodeRep* newIN (TableExprNodeRep* right,
+ const TaQLStyle&) const;
TableExprNodeRep* newOR (TableExprNodeRep* right) const;
TableExprNodeRep* newAND (TableExprNodeRep* right) const;
// </group>
@@ -1286,9 +1289,10 @@
{
return right.newGT (left.node_p);
}
-inline TableExprNode TableExprNode::in (const TableExprNode& right) const
+inline TableExprNode TableExprNode::in (const TableExprNode& right,
+ const TaQLStyle& style) const
{
- return newIN (right.node_p);
+ return newIN (right.node_p, style);
}
inline TableExprNode TableExprNode::operator() (const TableExprNodeSet&
indices)
{
=======================================
--- /branches/nov14/tables/TaQL/ExprNodeArray.cc Wed Dec 10 08:06:42 2014
UTC
+++ /branches/nov14/tables/TaQL/ExprNodeArray.cc Wed Mar 18 15:00:56 2015
UTC
@@ -61,6 +61,65 @@

TableExprNodeArray::~TableExprNodeArray()
{}
+
+TableExprNodeRep* TableExprNodeArray::makeConstantScalar()
+{
+ if (isConstant()) {
+ switch (dataType()) {
+ case NTBool:
+ {
+ Array<Bool> arr = getArrayBool(0);
+ if (arr.size() == 1) {
+ return new TableExprNodeConstBool (arr.data()[0]);
+ }
+ }
+ break;
+ case NTInt:
+ {
+ Array<Int64> arr = getArrayInt(0);
+ if (arr.size() == 1) {
+ return new TableExprNodeConstInt (arr.data()[0]);
+ }
+ }
+ break;
+ case NTDouble:
+ {
+ Array<Double> arr = getArrayDouble(0);
+ if (arr.size() == 1) {
+ return new TableExprNodeConstDouble (arr.data()[0]);
+ }
+ }
+ break;
+ case NTComplex:
+ {
+ Array<DComplex> arr = getArrayDComplex(0);
+ if (arr.size() == 1) {
+ return new TableExprNodeConstDComplex (arr.data()[0]);
+ }
+ }
+ break;
+ case NTString:
+ {
+ Array<String> arr = getArrayString(0);
+ if (arr.size() == 1) {
+ return new TableExprNodeConstString (arr.data()[0]);
+ }
+ }
+ break;
+ case NTDate:
+ {
+ Array<MVTime> arr = getArrayDate(0);
+ if (arr.size() == 1) {
+ return new TableExprNodeConstDate (arr.data()[0]);
+ }
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ return 0;
+}

const IPosition& TableExprNodeArray::getShape (const TableExprId& id)
{
=======================================
--- /branches/nov14/tables/TaQL/ExprNodeArray.h Wed Jan 7 09:08:57 2015 UTC
+++ /branches/nov14/tables/TaQL/ExprNodeArray.h Wed Mar 18 15:00:56 2015 UTC
@@ -75,6 +75,11 @@

~TableExprNodeArray();

+ // Turn a constant array with one element into a scalar.
+ // It returns a zero pointer if not possible.
+ // The default implementation returns 0.
+ virtual TableExprNodeRep* makeConstantScalar();
+
// Get the shape of the array in the given row.
// This default implementation evaluates the value and returns its
shape.
virtual const IPosition& getShape (const TableExprId& id);
=======================================
--- /branches/nov14/tables/TaQL/TaQLNodeHandler.cc Wed Feb 18 09:15:05 2015
UTC
+++ /branches/nov14/tables/TaQL/TaQLNodeHandler.cc Wed Mar 18 15:00:56 2015
UTC
@@ -202,7 +202,7 @@
case TaQLBinaryNodeRep::B_LE:
return new TaQLNodeHRValue (left <= right);
case TaQLBinaryNodeRep::B_IN:
- return new TaQLNodeHRValue (left.in (right));
+ return new TaQLNodeHRValue (left.in (right, node.style()));
case TaQLBinaryNodeRep::B_INDEX:
break;
case TaQLBinaryNodeRep::B_EQREGEX:
=======================================
--- /branches/nov14/tables/TaQL/UDFBase.h Wed Mar 18 09:03:37 2015 UTC
+++ /branches/nov14/tables/TaQL/UDFBase.h Wed Mar 18 15:00:56 2015 UTC
@@ -114,7 +114,7 @@
// <tr>
// <td><src>getXXX</src></td>
// <td>these are virtual get functions for each possible data type. The
- // get functions matching the data types that set by the setup
+ // get functions matching the data types set by the setup
// function need to be implemented.
// The <src>get</src> functions have an argument TableExprId
// defining the table row (or record) for which the function has