Updated overrides based on suggestions

Author: sdl60660

name_server/data_processing/aggregate_name_suggestions.js

@@ -71,7 +71,7 @@ const runAggregation = async (aggregationPipeline, collection) => {
 
         const coll = client.db("river_runner").collection(collection);
 
-        coll.aggregate(aggregationPipeline, async (cmdErr, result) => {
+        coll.aggregate(aggregationPipeline, { allowDiskUse: true, cursor: { batchSize: 1000 } }, async (cmdErr, result) => {
           assert.equal(null, cmdErr);
 
           const aggregationData = await result.toArray();
@@ -88,43 +88,43 @@ const main = async () => {
   const csvFilepath = "../../public/data/name_overrides.csv";
 
   const existingOverrides = await getExistingOverrides(csvFilepath);
-  const groupedOccurences = await runAggregation(unnamedFeatureCounts, "unnamed_features");
+  // const groupedOccurences = await runAggregation(unnamedFeatureCounts, "unnamed_features");
   const groupedSuggestions = await runAggregation(suggestionCounts, "suggestions");
 
-  //   const overridenIDs = existingOverrides.map((d) => Number(d.levelpathid));
-  //   const unhandledSuggestions = groupedSuggestions
-  //     .filter((d) => d.count > 3)
-  //     .filter((d) => !overridenIDs.includes(d._id));
-
-  //   console.log(unhandledSuggestions);
-  const joinedData = groupedOccurences.map((d) => {
-    const suggestions = groupedSuggestions.find((a) => a._id === d._id) || {
-      count: 0,
-      route_url: [],
-      suggestions: null,
-    };
-
-    const override = existingOverrides.find((a) => Number(a.levelpathid) === d._id);
-
-    return {
-      levelpathid: d._id,
-      sample_route_url: [...suggestions.route_url, d.route_url].filter((d) => d)[0],
-      num_suggestions: suggestions.count,
-      num_occurences: d.count,
-      suggestions: suggestions.suggestions,
-      overriden: override ? true : false,
-      overriden_val: override ? override.feature_name : null,
-    };
-  });
-
-  const outputFile = fs.createWriteStream("data/aggregated_suggestions.csv");
-
-  csv
-    .write(joinedData, { headers: true })
-    .on("finish", function () {
-      console.log("Write to CSV successfully!");
-    })
-    .pipe(outputFile);
+  const overridenIDs = existingOverrides.map((d) => Number(d.levelpathid));
+  const unhandledSuggestions = groupedSuggestions
+    .filter((d) => d.count > 3)
+    .filter((d) => !overridenIDs.includes(d._id));
+
+  console.log(unhandledSuggestions);
+  // const joinedData = groupedOccurences.map((d) => {
+  //   const suggestions = groupedSuggestions.find((a) => a._id === d._id) || {
+  //     count: 0,
+  //     route_url: [],
+  //     suggestions: null,
+  //   };
+
+  //   const override = existingOverrides.find((a) => Number(a.levelpathid) === d._id);
+
+  //   return {
+  //     levelpathid: d._id,
+  //     sample_route_url: [...suggestions.route_url, d.route_url].filter((d) => d)[0],
+  //     num_suggestions: suggestions.count,
+  //     num_occurences: d.count,
+  //     suggestions: suggestions.suggestions,
+  //     overriden: override ? true : false,
+  //     overriden_val: override ? override.feature_name : null,
+  //   };
+  // });
+
+  // const outputFile = fs.createWriteStream("data/aggregated_suggestions.csv");
+
+  // csv
+  //   .write(joinedData, { headers: true })
+  //   .on("finish", function () {
+  //     console.log("Write to CSV successfully!");
+  //   })
+  //   .pipe(outputFile);
 };
 
 main();

public/data/name_overrides.csv

@@ -220,4 +220,52 @@ levelpathid,feature_name
 1499364,Soar
 775504,Verde
 835801,Rosebud
-834198,Red Deer
+834198,Red Deer
+1496234,Berounka
+1504880,Töss
+1504556,Emme
+1504398,Nidda
+1416325,Drac
+1504319,Sieg
+1522090,Mondego
+1516793,Hers-Mort
+1516780,Aveyron
+1469298,Iller
+1416435,Ubaye
+1500212,Fulda
+1423150,Valira
+1496257,Sázava
+1504595,Amblève
+1467245,Danube
+863697,Thames
+1505211,Suhre
+1504371,Erft
+1416292,Ardèche
+1513466,Clain
+1505530,Lorze
+1468237,Iskar
+1508236,Wye
+1087803,Krokodil
+1468691,Lech
+1496836,Alster
+1468306,Salzach
+1562047,Eider
+1522591,Manzanares
+1494757,Tweed
+1500371,Werre
+1416446,Fier
+1422413,Francolí
+1565684,Stör
+1509367,Dijle
+1518333,Adour
+1504187,Lippe
+1504383,Thur
+1504240,Sambre
+1504778,Glatt
+1503876,Meuse
+1505020,Sitter
+1504777,Birs
+1504847,Vesdre
+1504653,Semois
+1504255,Reuss
+1504091,Rhein