Handle null values in numeric histogram (apple#1326)

Previously, all null values were ignored. With this change, if there are any missing values (true None, not NaN), they get counted separately in a categorical bin that is separated from the X axis. This change does not yet account for NaN/inf values. Those are still being ignored.
alexpsotelo · Jan 28, 2019 · d6fdd4d · d6fdd4d
1 parent 23ed996
commit d6fdd4d
Show file tree

Hide file tree

Showing 2 changed files with 85 additions and 6 deletions.
diff --git a/src/unity/lib/visualization/histogram.cpp b/src/unity/lib/visualization/histogram.cpp
@@ -295,6 +295,9 @@ std::string histogram_result::vega_column_data(bool) const {
   std::stringstream ss;
 
   for (size_t i=0; i<bins.bins.size(); i++) {
+    if (i != 0) {
+      ss << ",";
+    }
     const auto& value = bins.bins[i];
     ss << "{\"left\": ";
     ss << (bins.min + (i * binWidth));
@@ -303,9 +306,16 @@ std::string histogram_result::vega_column_data(bool) const {
     ss << ", \"count\": ";
     ss << value;
     ss << "}";
-    if (i != bins.bins.size() - 1) {
-      ss << ",";
-    }
+  }
+
+  // if there are null values, include them separately
+  size_t null_count = m_count.emit() - m_non_null_count.emit();
+  if (null_count > 0) {
+    ss << ",";
+    ss << "{\"missing\": true";
+    ss << ", \"count\": ";
+    ss << null_count;
+    ss << "}";
   }
 
   return ss.str();

diff --git a/src/unity/lib/visualization/vega_spec/histogram.json b/src/unity/lib/visualization/vega_spec/histogram.json
@@ -7,6 +7,16 @@
   "padding": 8,
   "title": {{title}},
   "style": "cell",
+  "signals": [
+    {
+      "name": "nullGap",
+      "update": "data('nulls').length > 0 ? 10 : 0"
+    },
+    {
+      "name": "barStep",
+      "update": "nullGap == 0 ? 0 : (width/40)"
+    }
+  ],
   "data": [
     {
       "name": "source_2"
@@ -35,6 +45,21 @@
           "expr": "datum[\"left\"] !== null && !isNaN(datum[\"left\"]) && datum[\"count\"] !== null && !isNaN(datum[\"count\"])"
         }
       ]
+    },
+    {
+      "source": "source_2",
+      "name": "nulls",
+      "transform": [
+        {
+          "expr": "toNumber(datum[\"count\"])",
+          "as": "count",
+          "type": "formula"
+        },
+        {
+          "expr": "datum[\"missing\"] === true && datum[\"count\"] !== null && !isNaN(datum[\"count\"])",
+          "type": "filter"
+        }
+      ]
     }
   ],
   "marks": [
@@ -75,9 +100,41 @@
           }
         }
       }
+    },
+    {
+      "type": "rect",
+      "from": {"data": "nulls"},
+      "style": [
+        "rect"
+      ],
+      "encode": {
+        "update": {
+          "x": {"scale": "xscale-null", "value": null, "offset": 1},
+          "x2": {"scale": "xscale-null", "band": 1},
+          "y": {"scale": "y", "field": "count"},
+          "y2": {"scale": "y", "value": 0},
+          "fill": {"value": "#108EE9"}
+        },
+        "hover": {
+          "fill": {"value": "#7EC2F3"}
+        }
+      }
     }
   ],
   "scales": [
+    {
+      "domain": [
+        null
+      ],
+      "range": [
+        0,
+        {
+          "signal": "barStep"
+        }
+      ],
+      "type": "band",
+      "name": "xscale-null"
+    },
     {
       "name": "x",
       "type": "linear",
@@ -90,7 +147,9 @@
         "sort": true
       },
       "range": [
-        0,
+        {
+          "signal": "barStep + nullGap"
+        },
         {
           "signal": "width"
         }
@@ -102,8 +161,10 @@
       "name": "y",
       "type": "linear",
       "domain": {
-        "data": "data_0",
-        "field": "count"
+        "fields": [
+          {"data": "data_0", "field": "count"},
+          {"data": "nulls", "field": "count"}
+        ]
       },
       "range": [
         {
@@ -165,6 +226,14 @@
       "ticks": false,
       "zindex": 0,
       "gridScale": "x"
+    },
+    {
+      "orient": "bottom",
+      "scale": "xscale-null",
+      "encode": {
+        "ticks": {"update": {"opacity": {"signal": "nullGap > 0 ? 1.0 : 0.0"}}},
+        "labels": {"update": {"text": {"signal": "nullGap > 0 ? 'null' : ''"}}}
+      }
     }
   ],
   {{config}}