Improvements to EXPLAIN QUERY PLAN formatting. The MULTI-INDEX OR now…

… shows

a separate "INDEX" subtree for each index.  SCALAR SUBQUERY entries provide
a subquery number that is related back to the .selecttrace output.

ext/expert/expert1.test

@@ -243,8 +243,10 @@ do_setup_rec_test $tn.12.1 {
   CREATE INDEX t7_idx_00000062 ON t7(b);
   CREATE INDEX t7_idx_00000061 ON t7(a);
   MULTI-INDEX OR
-    SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
-    SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
+    INDEX 1
+      SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
+    INDEX 2
+      SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
 }
 
 # rowid terms.

src/expr.c

@@ -2691,10 +2691,15 @@ void sqlite3CodeRhsOfIN(
     ** might not have been invoked yet, so invoke it now as a subroutine. 
     */
     if( ExprHasProperty(pExpr, EP_Subrtn) ){
-      sqlite3VdbeAddOp2(v, OP_Once, 0, sqlite3VdbeCurrentAddr(v)+3);
+      int addr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
+      if( ExprHasProperty(pExpr, EP_xIsSelect) ){
+        ExplainQueryPlan((pParse, 0, "REUSE LIST SUBQUERY %d",
+              pExpr->x.pSelect->selId));
+      }
       sqlite3VdbeAddOp2(v, OP_Gosub, pExpr->y.sub.regReturn,
                         pExpr->y.sub.iAddr);
       sqlite3VdbeAddOp2(v, OP_OpenDup, iTab, pExpr->iTable);
+      sqlite3VdbeJumpHere(v, addr);
       return;
     }
 
@@ -2746,7 +2751,7 @@ void sqlite3CodeRhsOfIN(
     if( ALWAYS(pEList->nExpr==nVal) ){
       SelectDest dest;
       int i;
-      sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable);
+      sqlite3SelectDestInit(&dest, SRT_Set, iTab);
       dest.zAffSdst = exprINAffinity(pParse, pExpr);
       pSelect->iLimit = 0;
       testcase( pSelect->selFlags & SF_Distinct );
@@ -2810,17 +2815,17 @@ void sqlite3CodeRhsOfIN(
 
       /* Evaluate the expression and insert it into the temp table */
       if( isRowid && sqlite3ExprIsInteger(pE2, &iValToIns) ){
-        sqlite3VdbeAddOp3(v, OP_InsertInt, pExpr->iTable, r2, iValToIns);
+        sqlite3VdbeAddOp3(v, OP_InsertInt, iTab, r2, iValToIns);
       }else{
         r3 = sqlite3ExprCodeTarget(pParse, pE2, r1);
         if( isRowid ){
           sqlite3VdbeAddOp2(v, OP_MustBeInt, r3,
                             sqlite3VdbeCurrentAddr(v)+2);
           VdbeCoverage(v);
-          sqlite3VdbeAddOp3(v, OP_Insert, pExpr->iTable, r2, r3);
+          sqlite3VdbeAddOp3(v, OP_Insert, iTab, r2, r3);
         }else{
           sqlite3VdbeAddOp4(v, OP_MakeRecord, r3, 1, r2, &affinity, 1);
-          sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pExpr->iTable, r2, r3, 1);
+          sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r2, r3, 1);
         }
       }
     }
@@ -2864,6 +2869,11 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){
 
   Vdbe *v = pParse->pVdbe;
   assert( v!=0 );
+  testcase( pExpr->op==TK_EXISTS );
+  testcase( pExpr->op==TK_SELECT );
+  assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT );
+  assert( ExprHasProperty(pExpr, EP_xIsSelect) );
+  pSel = pExpr->x.pSelect;
 
   /* The evaluation of the EXISTS/SELECT must be repeated every time it
   ** is encountered if any of the following is true:
@@ -2879,6 +2889,7 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){
     /* If this routine has already been coded, then invoke it as a
     ** subroutine. */
     if( ExprHasProperty(pExpr, EP_Subrtn) ){
+      ExplainQueryPlan((pParse, 0, "REUSE SUBQUERY %d", pSel->selId));
       sqlite3VdbeAddOp2(v, OP_Gosub, pExpr->y.sub.regReturn,
                         pExpr->y.sub.iAddr);
       return pExpr->iTable;
@@ -2904,14 +2915,8 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){
   ** In both cases, the query is augmented with "LIMIT 1".  Any 
   ** preexisting limit is discarded in place of the new LIMIT 1.
   */
-  testcase( pExpr->op==TK_EXISTS );
-  testcase( pExpr->op==TK_SELECT );
-  assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT );
-  assert( ExprHasProperty(pExpr, EP_xIsSelect) );
-
-  pSel = pExpr->x.pSelect;
-  ExplainQueryPlan((pParse, 1, "%sSCALAR SUBQUERY",
-        addrOnce?"":"CORRELATED "));
+  ExplainQueryPlan((pParse, 1, "%sSCALAR SUBQUERY %d",
+        addrOnce?"":"CORRELATED ", pSel->selId));
   nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1;
   sqlite3SelectDestInit(&dest, 0, pParse->nMem+1);
   pParse->nMem += nReg;

src/vdbe.h

@@ -208,13 +208,20 @@ VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp,int iLineno);
   void sqlite3VdbeExplain(Parse*,u8,const char*,...);
   void sqlite3VdbeExplainPop(Parse*);
   int sqlite3VdbeExplainParent(Parse*);
+  void sqlite3ExplainBreakpoint(const char*,const char*);
 # define ExplainQueryPlan(P)        sqlite3VdbeExplain P
 # define ExplainQueryPlanPop(P)     sqlite3VdbeExplainPop(P)
 # define ExplainQueryPlanParent(P)  sqlite3VdbeExplainParent(P)
 #else
 # define ExplainQueryPlan(P)
 # define ExplainQueryPlanPop(P)
 # define ExplainQueryPlanParent(P) 0
+# define sqlite3ExplainBreakpoint(A,B) /*no-op*/
+#endif
+#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_EXPLAIN)
+  void sqlite3ExplainBreakpoint(const char*,const char*);
+#else
+# define sqlite3ExplainBreakpoint(A,B) /*no-op*/
 #endif
 void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*);
 void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8);

src/vdbeaux.c

@@ -350,13 +350,27 @@ int sqlite3VdbeExplainParent(Parse *pParse){
 }
 
 /*
-** Add a new OP_Explain opcode.
+** Set a debugger breakpoint on the following routine in order to
+** monitor the EXPLAIN QUERY PLAN code generation.
+*/
+#if defined(SQLITE_DEBUG)
+void sqlite3ExplainBreakpoint(const char *z1, const char *z2){
+  (void)z1;
+  (void)z2;
+}
+#endif
+
+/*
+** Add a new OP_ opcode.
 **
 ** If the bPush flag is true, then make this opcode the parent for
 ** subsequent Explains until sqlite3VdbeExplainPop() is called.
 */
 void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){
-  if( pParse->explain==2 ){
+#if !defined(SQLITE_DEBUG)
+  if( pParse->explain==2 )
+#endif
+  {
     char *zMsg;
     Vdbe *v;
     va_list ap;
@@ -368,14 +382,18 @@ void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){
     iThis = v->nOp;
     sqlite3VdbeAddOp4(v, OP_Explain, iThis, pParse->addrExplain, 0,
                       zMsg, P4_DYNAMIC);
-    if( bPush) pParse->addrExplain = iThis;
+    sqlite3ExplainBreakpoint(bPush?"PUSH":"", sqlite3VdbeGetOp(v,-1)->p4.z);
+    if( bPush){
+      pParse->addrExplain = iThis;
+    }
   }
 }
 
 /*
 ** Pop the EXPLAIN QUERY PLAN stack one level.
 */
 void sqlite3VdbeExplainPop(Parse *pParse){
+  sqlite3ExplainBreakpoint("POP", 0);
   pParse->addrExplain = sqlite3VdbeExplainParent(pParse);
 }
 #endif /* SQLITE_OMIT_EXPLAIN */

src/wherecode.c

@@ -213,6 +213,7 @@ int sqlite3WhereExplainOneScan(
     }
 #endif
     zMsg = sqlite3StrAccumFinish(&str);
+    sqlite3ExplainBreakpoint("",zMsg);
     ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v),
                             pParse->addrExplain, 0, zMsg,P4_DYNAMIC);
   }
@@ -1951,6 +1952,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
           pOrExpr = pAndExpr;
         }
         /* Loop through table entries that match term pOrTerm. */
+        ExplainQueryPlan((pParse, 1, "INDEX %d", ii+1));
         WHERETRACE(0xffff, ("Subplan for OR-clause:\n"));
         pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0,
                                       wctrlFlags, iCovCur);
@@ -2054,6 +2056,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
 
           /* Finish the loop through table entries that match term pOrTerm. */
           sqlite3WhereEnd(pSubWInfo);
+          ExplainQueryPlanPop(pParse);
         }
       }
     }

test/autoindex1.test

@@ -280,7 +280,7 @@ do_eqp_test autoindex1-600a {
   |--MATERIALIZE xxxxxx
   |  |--SCAN TABLE sheep AS s
   |  |--SEARCH TABLE flock_owner AS prev USING INDEX sqlite_autoindex_flock_owner_1 (flock_no=? AND owner_change_date<?)
-  |  `--CORRELATED SCALAR SUBQUERY
+  |  `--CORRELATED SCALAR SUBQUERY xxxxxx
   |     `--SEARCH TABLE flock_owner AS later USING COVERING INDEX sqlite_autoindex_flock_owner_1 (flock_no=? AND owner_change_date>? AND owner_change_date<?)
   |--SCAN TABLE sheep AS x USING INDEX sheep_reg_flock_index
   `--SEARCH SUBQUERY xxxxxx AS y USING AUTOMATIC COVERING INDEX (sheep_no=?)

test/bestindex3.test

@@ -90,17 +90,21 @@ do_eqp_test 1.3 {
 } {
   QUERY PLAN
   `--MULTI-INDEX OR
-     |--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a EQ ?
-     `--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?
+     |--INDEX 1
+     |  `--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a EQ ?
+     `--INDEX 2
+        `--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?
 }
 
 do_eqp_test 1.4 {
   SELECT * FROM t1 WHERE a LIKE 'abc%' OR b = 'def';
 } {
   QUERY PLAN
   `--MULTI-INDEX OR
-     |--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a LIKE ?
-     `--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?
+     |--INDEX 1
+     |  `--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a LIKE ?
+     `--INDEX 2
+        `--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?
 }
 
 do_execsql_test 1.5 {
@@ -150,8 +154,10 @@ ifcapable !icu {
   } [string map {"\n  " \n} {
     QUERY PLAN
     `--MULTI-INDEX OR
-       |--SEARCH TABLE t2 USING INDEX t2x (x>? AND x<?)
-       `--SEARCH TABLE t2 USING INDEX t2y (y=?)
+       |--INDEX 1
+       |  `--SEARCH TABLE t2 USING INDEX t2x (x>? AND x<?)
+       `--INDEX 2
+          `--SEARCH TABLE t2 USING INDEX t2y (y=?)
   }]
 }
 

test/cost.test

@@ -58,9 +58,12 @@ do_eqp_test 3.2 {
 } {
   QUERY PLAN
   |--MULTI-INDEX OR
-  |  |--SEARCH TABLE t5 USING INDEX t5b (b=?)
-  |  |--SEARCH TABLE t5 USING INDEX t5c (c=?)
-  |  `--SEARCH TABLE t5 USING INDEX t5d (d=?)
+  |  |--INDEX 1
+  |  |  `--SEARCH TABLE t5 USING INDEX t5b (b=?)
+  |  |--INDEX 2
+  |  |  `--SEARCH TABLE t5 USING INDEX t5c (c=?)
+  |  `--INDEX 3
+  |     `--SEARCH TABLE t5 USING INDEX t5d (d=?)
   `--USE TEMP B-TREE FOR ORDER BY
 }
 
@@ -124,8 +127,10 @@ do_eqp_test 6.2 {
 } {
   QUERY PLAN
   |--MULTI-INDEX OR
-  |  |--SEARCH TABLE t3 USING INDEX t3i1 (b>? AND b<?)
-  |  `--SEARCH TABLE t3 USING INDEX t3i2 (c=?)
+  |  |--INDEX 1
+  |  |  `--SEARCH TABLE t3 USING INDEX t3i1 (b>? AND b<?)
+  |  `--INDEX 2
+  |     `--SEARCH TABLE t3 USING INDEX t3i2 (c=?)
   `--USE TEMP B-TREE FOR ORDER BY
 }
 
@@ -149,8 +154,10 @@ do_eqp_test 7.2 {
 } {
   QUERY PLAN
   |--MULTI-INDEX OR
-  |  |--SEARCH TABLE t1 USING INDEX t1b (b>? AND b<?)
-  |  `--SEARCH TABLE t1 USING INDEX t1b (b=?)
+  |  |--INDEX 1
+  |  |  `--SEARCH TABLE t1 USING INDEX t1b (b>? AND b<?)
+  |  `--INDEX 2
+  |     `--SEARCH TABLE t1 USING INDEX t1b (b=?)
   `--USE TEMP B-TREE FOR ORDER BY
 }
 

test/eqp.test

@@ -45,8 +45,10 @@ do_eqp_test 1.2 {
 } {
   QUERY PLAN
   |--MULTI-INDEX OR
-  |  |--SEARCH TABLE t1 USING INDEX i1 (a=?)
-  |  `--SEARCH TABLE t1 USING INDEX i2 (b=?)
+  |  |--INDEX 1
+  |  |  `--SEARCH TABLE t1 USING INDEX i1 (a=?)
+  |  `--INDEX 2
+  |     `--SEARCH TABLE t1 USING INDEX i2 (b=?)
   `--SCAN TABLE t2
 }
 do_eqp_test 1.3 {
@@ -55,8 +57,10 @@ do_eqp_test 1.3 {
   QUERY PLAN
   |--SCAN TABLE t2
   `--MULTI-INDEX OR
-     |--SEARCH TABLE t1 USING INDEX i1 (a=?)
-     `--SEARCH TABLE t1 USING INDEX i2 (b=?)
+     |--INDEX 1
+     |  `--SEARCH TABLE t1 USING INDEX i1 (a=?)
+     `--INDEX 2
+        `--SEARCH TABLE t1 USING INDEX i2 (b=?)
 }
 do_eqp_test 1.3 {
   SELECT a FROM t1 ORDER BY a
@@ -225,23 +229,23 @@ do_eqp_test 3.1.1 {
 } {
   QUERY PLAN
   |--SCAN TABLE t1
-  `--SCALAR SUBQUERY
+  `--SCALAR SUBQUERY xxxxxx
      `--SCAN TABLE t1 AS sub
 }
 do_eqp_test 3.1.2 {
   SELECT * FROM t1 WHERE (SELECT x FROM t1 AS sub);
 } {
   QUERY PLAN
   |--SCAN TABLE t1
-  `--SCALAR SUBQUERY
+  `--SCALAR SUBQUERY xxxxxx
      `--SCAN TABLE t1 AS sub
 }
 do_eqp_test 3.1.3 {
   SELECT * FROM t1 WHERE (SELECT x FROM t1 AS sub ORDER BY y);
 } {
   QUERY PLAN
   |--SCAN TABLE t1
-  `--SCALAR SUBQUERY
+  `--SCALAR SUBQUERY xxxxxx
      |--SCAN TABLE t1 AS sub
      `--USE TEMP B-TREE FOR ORDER BY
 }
@@ -250,7 +254,7 @@ do_eqp_test 3.1.4 {
 } {
   QUERY PLAN
   |--SCAN TABLE t1
-  `--SCALAR SUBQUERY
+  `--SCALAR SUBQUERY xxxxxx
      `--SCAN TABLE t2 USING COVERING INDEX t2i1
 }
 
@@ -302,7 +306,7 @@ det 3.3.3 {
 } {
   QUERY PLAN
   |--SCAN TABLE t1
-  `--CORRELATED SCALAR SUBQUERY
+  `--CORRELATED SCALAR SUBQUERY xxxxxx
      `--SCAN TABLE t2
 }
 
@@ -813,7 +817,7 @@ do_eqp_test 9.1 {
   |--MATERIALIZE xxxxxx
   |  |--SCAN TABLE forumpost AS x USING INDEX forumthread
   |  |--USING ROWID SEARCH ON TABLE private FOR IN-OPERATOR
-  |  |--CORRELATED SCALAR SUBQUERY
+  |  |--CORRELATED SCALAR SUBQUERY xxxxxx
   |  |  |--SEARCH TABLE forumpost USING COVERING INDEX forumthread (froot=?)
   |  |  `--USING ROWID SEARCH ON TABLE private FOR IN-OPERATOR
   |  `--USE TEMP B-TREE FOR ORDER BY

test/join5.test

@@ -267,8 +267,10 @@ do_eqp_test 7.2 {
   QUERY PLAN
   |--SCAN TABLE t1
   `--MULTI-INDEX OR
-     |--SEARCH TABLE t2 USING INDEX t2xy (x=? AND y=?)
-     `--SEARCH TABLE t2 USING INDEX t2xy (x=? AND y=?)
+     |--INDEX 1
+     |  `--SEARCH TABLE t2 USING INDEX t2xy (x=? AND y=?)
+     `--INDEX 2
+        `--SEARCH TABLE t2 USING INDEX t2xy (x=? AND y=?)
 }
 
 do_execsql_test 7.3 {

test/tkt-80ba201079.test

@@ -110,6 +110,8 @@ do_test tkt-80ba2-200 {
 } {300 object_change 2048}
 do_test tkt-80ba2-201 {
   db eval {
+PRAGMA vdbe_debug=on;
+PRAGMA vdbe_addoptrace=on;
     CREATE INDEX timeline_entry_id_idx on timeline(entry_id);
     SELECT entry_type,
            entry_types.name,

test/where7.test

@@ -23353,8 +23353,10 @@ do_eqp_test where7-3.2 {
 } {
   QUERY PLAN
   |--MULTI-INDEX OR
-  |  |--SEARCH TABLE t301 USING COVERING INDEX t301_c4 (c4=?)
-  |  `--SEARCH TABLE t301 USING INTEGER PRIMARY KEY (rowid=?)
+  |  |--INDEX 1
+  |  |  `--SEARCH TABLE t301 USING COVERING INDEX t301_c4 (c4=?)
+  |  `--INDEX 2
+  |     `--SEARCH TABLE t301 USING INTEGER PRIMARY KEY (rowid=?)
   |--SEARCH TABLE t302 USING INDEX t302_c8_c3 (c8=? AND c3>?)
   `--USE TEMP B-TREE FOR ORDER BY
 }