more complete refactor of loading Foamcore rows more optimistically

- also a rendering optimization for MovableGridCard, it renders a 
simpler GridCardPreload (gray square) before trying to render the full 
card

Author: kaptron

app/javascript/stores/RoutingStore.js

@@ -133,6 +133,7 @@ class RoutingStore extends RouterStore {
     this.push(path)
   }
 
+  @action
   beforeRouting() {
     const { uiStore } = this
     // close the org/roles menus if either are open when we route to a new page

app/javascript/stores/UiStore.js

@@ -100,6 +100,8 @@ export default class UiStore {
   @observable
   isLoading = false
   @observable
+  isRouting = false
+  @observable
   isTransparentLoading = false
   @observable
   isLoadingMoveAction = false

app/javascript/stores/jsonApi/Collection.js

@@ -22,12 +22,15 @@ import SharedRecordMixin from './SharedRecordMixin'
 import v, { FOAMCORE_MAX_ZOOM, FOUR_WIDE_MAX_ZOOM } from '~/utils/variables'
 import { POPUP_ACTION_TYPES } from '~/enums/actionEnums'
 import { methodLibraryTags } from '~/utils/creativeDifferenceVariables'
+import { objectsEqual } from '~/utils/objectUtils'
 
 export const ROW_ACTIONS = {
   INSERT: 'insert_row',
   REMOVE: 'remove_row',
 }
 
+const CARD_PROPERTIES = ['id', 'updated_at', 'order', 'row', 'col']
+
 class Collection extends SharedRecordMixin(BaseRecord) {
   static type = 'collections'
   static endpoint = apiUrl('collections')
@@ -621,9 +624,16 @@ class Collection extends SharedRecordMixin(BaseRecord) {
 
   @computed
   get cardProperties() {
-    return this.collection_cards.map(c =>
-      _.pick(c, ['id', 'updated_at', 'order', 'row', 'col'])
-    )
+    return this.collection_cards.map(c => _.pick(c, CARD_PROPERTIES))
+  }
+
+  @action
+  replaceCardsIfDifferent(newCards) {
+    const newProperties = newCards.map(c => _.pick(c, CARD_PROPERTIES))
+    if (objectsEqual(newProperties, this.cardProperties)) {
+      return
+    }
+    this.collection_cards.replace(newCards)
   }
 
   get allowsCollectionTypeSelector() {
@@ -840,6 +850,7 @@ class Collection extends SharedRecordMixin(BaseRecord) {
         this.currentOrder = order
       }
     })
+    const { uiStore } = this
     const params = {
       page,
       per_page,
@@ -861,6 +872,7 @@ class Collection extends SharedRecordMixin(BaseRecord) {
     }
     let apiPath
     if (searchTerm) {
+      uiStore.update('isTransparentLoading', true)
       params.query = searchTerm
       if (this.collectionFilterQuery.q) {
         params.query += ` ${this.collectionFilterQuery.q}`
@@ -886,12 +898,14 @@ class Collection extends SharedRecordMixin(BaseRecord) {
     const res = await this.apiStore.request(apiPath)
     const { data, links, meta } = res
     runInAction(() => {
+      uiStore.update('isTransparentLoading', false)
       if (searchTerm) {
         this.totalPages = (meta && meta.total_pages) || 1
       } else {
         this.totalPages = links.last
       }
-      const firstPage = page === 1 && (!rows || rows[0] === 0)
+      // NOTE: firstPage doesn't happen when loading rows
+      const firstPage = page === 1
       if (
         firstPage &&
         (this.storedCacheKey !== this.cache_key ||
@@ -910,16 +924,7 @@ class Collection extends SharedRecordMixin(BaseRecord) {
         if (this.currentPage < page) {
           this.currentPage = page
         }
-        const newData = _.reverse(
-          // de-dupe merged data (deferring to new cards first)
-          // reverse + reverse so that new cards (e.g. page 2) are replaced first but then put back at the end
-          _.unionBy(
-            _.reverse([...data]),
-            _.reverse([...this.collection_cards]),
-            'id'
-          )
-        )
-        this.collection_cards.replace(newData)
+        this.mergeCards(data)
       }
 
       if (this.isSplitLevelBottom) {
@@ -959,9 +964,16 @@ class Collection extends SharedRecordMixin(BaseRecord) {
 
   @action
   mergeCards = cards => {
-    // de-dupe merged data (deferring to new cards first)
-    const newData = _.unionBy(cards, this.collection_cards, 'id')
-    this.collection_cards.replace(_.sortBy(newData, 'order'))
+    const newData = _.reverse(
+      // de-dupe merged data (deferring to new cards first)
+      // reverse + reverse so that new cards (e.g. page 2) are replaced first but then put back at the end
+      _.unionBy(
+        _.reverse([...cards]),
+        _.reverse([...this.collection_cards]),
+        'id'
+      )
+    )
+    this.collection_cards.replace(newData)
   }
 
   API_fetchCardOrders = async () => {

app/javascript/stores/jsonApi/Item.js

@@ -1,5 +1,5 @@
 import _ from 'lodash'
-import { observable, runInAction } from 'mobx'
+import { computed, observable, runInAction } from 'mobx'
 import { ReferenceType } from 'datx'
 
 import { apiUrl } from '~/utils/url'
@@ -206,9 +206,19 @@ class Item extends SharedRecordMixin(BaseRecord) {
     })
   }
 
+  @computed
   get primaryDataset() {
-    const { datasets } = this
-    if (!datasets) return null
+    const { apiStore } = this
+    let { datasets } = this
+    if (!datasets || datasets.length === 0) {
+      // look in apiStore for any datasets that we set ds.item === this
+      datasets = apiStore.findAll('datasets').filter(ds => {
+        return ds.item === this
+      })
+      if (!datasets) {
+        return null
+      }
+    }
     if (datasets.length === 1) return datasets[0]
     const primary = datasets.find(dataset => dataset.order === 0)
     return primary
@@ -333,7 +343,12 @@ class Item extends SharedRecordMixin(BaseRecord) {
     const res = await this.apiStore.request(`items/${this.id}/datasets`)
     runInAction(() => {
       this.loadingDatasets = false
-      this.datasets = res.data
+      const datasets = res.data
+      _.each(datasets, d => {
+        // set the dataset item reference in case we need to look it up later
+        d.item = this
+      })
+      this.datasets = datasets
     })
     return this.datasets
   }

app/javascript/ui/Routes.js

@@ -116,6 +116,15 @@ class Routes extends React.Component {
     })
   }
 
+  componentDidUpdate(prevProps) {
+    const { uiStore } = this.props
+    if (this.props.location !== prevProps.location) {
+      uiStore.update('isRouting', true)
+    } else if (uiStore.isRouting) {
+      uiStore.update('isRouting', false)
+    }
+  }
+
   componentWillUnmount() {
     document.removeEventListener('keydown', captureGlobalKeypress)
     document.removeEventListener('touchmove', this.handleTouchMove, {

app/javascript/ui/grid/FoamcoreGrid.js

@@ -90,6 +90,7 @@ class FoamcoreGrid extends React.Component {
     const { collection, uiStore } = this.props
     if (collection.id !== prevProps.collection.id) {
       uiStore.determineZoomLevels(collection)
+      this.loadInitialRows()
     }
 
     if (!objectsEqual(this.props.cardProperties, prevProps.cardProperties)) {
@@ -141,9 +142,8 @@ class FoamcoreGrid extends React.Component {
       this.handleZoomIn()
     }
 
-    // loadMoreRows via scrolling is just for infinite scroll / splitLevelBottom
     const visRows = uiStore.visibleRows
-    if (!collection.isSplitLevelBottom || !visRows) {
+    if (!visRows) {
       return
     }
 
@@ -155,31 +155,44 @@ class FoamcoreGrid extends React.Component {
     }
   }
 
-  loadInitialRows = () => {
+  loadInitialRows = async () => {
     const { collection, loadCollectionCards } = this.props
-    const { loadedRows } = collection
-    // const { loadMoreCollectionCards } = this
+    if (collection.isSplitLevelBottom) {
+      // this is only used for "normal" grids
+      return
+    }
+
     // arbitrary 300 row initial limit?
     const maxRow = _.min([collection.max_row_index, 300])
 
     const rowsPerPage = 30
-    let min = loadedRows
-    let max = loadedRows + rowsPerPage
-    console.log({ maxRow })
+    let min = 0
+    let max = rowsPerPage
+    const requests = []
     while (min <= maxRow) {
       // just fire off multiple async requests (without awaiting)
-      console.log('loadCollectionCards', [min, max])
-      loadCollectionCards({
-        // just load by row # downward, and always load all 16 cols
-        rows: [min, max],
+      const promise = new Promise(resolve => {
+        const rows = [min, max]
+        return resolve(
+          loadCollectionCards({
+            // just load by row # downward, and always load all 16 cols
+            rows,
+          })
+        )
       })
+      requests.push(promise)
       min = max + 1
       max = max + rowsPerPage
     }
+    const data = await Promise.all(requests)
+    runInAction(() => {
+      const newCards = _.flatten(data)
+      collection.replaceCardsIfDifferent(newCards)
+    })
   }
 
   loadMoreRows = () => {
-    const { collection } = this.props
+    const { collection, uiStore } = this.props
     const { loadMoreCollectionCards } = this
     if (collection.isSplitLevelBottom) {
       if (collection.hasMore) {
@@ -188,22 +201,23 @@ class FoamcoreGrid extends React.Component {
       return
     }
 
-    // const visRows = uiStore.visibleRows
-    // const collectionMaxRow = collection.max_row_index
-    // // min row should start with the next row after what's loaded
-    // const loadMinRow = collection.loadedRows + 1
-    // // add a buffer of 3 more rows (constrained by max row on collection)
-    // const loadMaxRow = _.min([
-    //   collectionMaxRow,
-    //   Math.ceil(loadMinRow + visRows.num + 3),
-    // ])
-    // // min and max could be equal if there is one more row to load
-    // if (loadMinRow <= loadMaxRow) {
-    //   return loadMoreCollectionCards({
-    //     // just load by row # downward, and always load all 16 cols
-    //     rows: [loadMinRow, loadMaxRow],
-    //   })
-    // }
+    // NOTE: you should really only get into here if you go past row 300+
+    const visRows = uiStore.visibleRows
+    const collectionMaxRow = collection.max_row_index
+    // min row should start with the next row after what's loaded
+    const loadMinRow = collection.loadedRows + 1
+    // add a buffer of 3 more rows (constrained by max row on collection)
+    const loadMaxRow = _.min([
+      collectionMaxRow,
+      Math.ceil(loadMinRow + visRows.num + 3),
+    ])
+    // min and max could be equal if there is one more row to load
+    if (loadMinRow <= loadMaxRow) {
+      return loadMoreCollectionCards({
+        // just load by row # downward, and always load all 16 cols
+        rows: [loadMinRow, loadMaxRow],
+      })
+    }
   }
 
   loadMoreCollectionCards = async (opts = {}) => {

app/javascript/ui/grid/MovableGridCard.js

@@ -19,6 +19,12 @@ import ResizeIcon from '~/ui/icons/ResizeIcon'
 import { StyledCardWrapper } from '~/ui/grid/shared'
 import { pageBoundsScroller } from '~/utils/ScrollNearPageBoundsService'
 
+const GridCardPreload = styled.div`
+  height: 100%;
+  width: 100%;
+  background: ${v.colors.commonMedium};
+`
+
 const StyledResizeIcon = styled.div`
   position: absolute;
   /* hide the resize icon while the menu is open so they don't overlap */
@@ -88,13 +94,26 @@ class MovableGridCard extends React.Component {
       resizeWidth: 0,
       resizeHeight: 0,
       allowTouchDeviceDragging: false,
+      // if we're mid-routing change, then set to preloading gray square
+      preloading: uiStore.isRouting,
     }
     this.debouncedAllowTouchDeviceDrag = _.debounce(() => {
       if (this.unmounted) return
       this.setState({ allowTouchDeviceDragging: true })
     }, v.touchDeviceHoldToDragTime)
   }
 
+  componentDidMount() {
+    if (!this.state.preloading) {
+      return
+    }
+    setTimeout(() => {
+      if (this.unmounted) return
+      // after a slight delay, turn preloading off and render the actual GridCard
+      this.setState({ preloading: false })
+    }, 150)
+  }
+
   componentDidUpdate(prevProps) {
     if (this.state.dragging || this.unmounted) {
       return
@@ -457,6 +476,7 @@ class MovableGridCard extends React.Component {
       resizeHeight,
       x,
       y,
+      preloading,
     } = this.state
 
     const {
@@ -564,7 +584,9 @@ class MovableGridCard extends React.Component {
     const adjustedWidth = (width + resizeWidth) / zoomLevel
     const adjustedHeight = (height + resizeHeight) / zoomLevel
     let transition =
-      dragging || resizing || currentlyZooming ? 'none' : cardCSSTransition
+      dragging || resizing || currentlyZooming || preloading
+        ? 'none'
+        : cardCSSTransition
     // TODO this should actually check it's a breadcrumb
     const draggedOverBreadcrumb = !!activeDragTarget
     if (dragging && this.state.allowTouchDeviceDragging) {
@@ -705,12 +727,16 @@ class MovableGridCard extends React.Component {
             transform={transform}
             zoomLevel={zoomLevel}
           >
-            <GridCard
-              {...cardProps}
-              draggingMultiple={draggingMultiple}
-              hoveringOver={hoveringOverRight}
-              zoomLevel={zoomLevel}
-            />
+            {/* During preload we just render a gray square to simplify initial render */}
+            {preloading && <GridCardPreload />}
+            {!preloading && (
+              <GridCard
+                {...cardProps}
+                draggingMultiple={draggingMultiple}
+                hoveringOver={hoveringOverRight}
+                zoomLevel={zoomLevel}
+              />
+            )}
           </InnerCardWrapper>
         </Rnd>
       </StyledCardWrapper>

app/javascript/ui/grid/covers/DataItemCover.js

@@ -30,10 +30,10 @@ class DataItemCover extends React.Component {
     }
   }
 
-  async loadDatasets() {
+  loadDatasets() {
     const { item } = this.props
     if (!item.loadingDatasets) {
-      await item.API_fetchDatasets()
+      item.API_fetchDatasets()
     }
   }