Ad data explorer

package.json

@@ -26,6 +26,8 @@
     "copy-webpack-plugin": "4.2.0",
     "core-js": "^3.8.0",
     "create-react-class": "^15.7.0",
+    "d3-fetch": "^2.0.0",
+    "d3-scale": "^3.2.3",
     "date-fns": "^2.23.0",
     "dateformat": "3.0.2",
     "eslint": "^7.14.0",
@@ -52,38 +54,52 @@
     "html-react-parser": "^0.14.2",
     "immutability-helper": "^2.6.6",
     "jquery": "^3.5.1",
+    "jsonexport": "^2.0.11",
+    "loader-utils": "1.0.2",
     "localforage": "^1.5.2",
     "marked": "^1.1.1",
     "matchmedia": "^0.1.2",
+    "moment": "2.20.1",
     "nib": "^1.1.2",
     "node-sass": "^4.14.1",
+    "pandas-js": "0.2.4",
     "pondjs": "0.8.7",
     "prismjs": "^1.22.0",
     "raw-loader": "^4.0.2",
+    "rc-checkbox": "2.1.4",
     "rc-slider": "8.3.1",
-    "react": "^16.4.2",
+    "react": "16.14.0",
     "react-autosuggest": "^9.3.2",
+    "react-basic-table": "^1.0.14",
     "react-chartjs-2": "^2.11.1",
+    "react-collapsible": "^2.0.4",
     "react-custom-scrollbars": "^4.2.1",
+    "react-datetime": "^2.14.0",
+    "react-day-picker": "^7.1.1",
     "react-document-title": "^2.0.3",
-    "react-dom": "^16.4.2",
+    "react-dom": "16.14.0",
     "react-helmet": "^5.1.3",
+    "react-highlight": "^0.14.0",
     "react-highlight.js": "^1.0.7",
     "react-joyride": "^2.0.5",
     "react-modal": "^3.1.0",
     "react-promise": "2.0.0",
     "react-radio-group": "^3.0.2",
-    "react-select": "2.4.4",
+    "react-select": "^4.3.0",
     "react-sticky-state": "^2.1.9",
-    "react-table": "6.9.0",
-    "react-timeseries-charts": "^0.14.0",
+    "react-table": "6.10.0",
+    "react-timeseries-charts": "^0.16.1",
     "react-tippy": "^1.2.3",
+    "react-tooltip": "^3.9.2",
     "react-transition-group": "^2.2.1",
     "reactstrap": "^7.1.0",
     "recharts": "1.8.5",
     "scroll-into-view": "^1.9.1",
+    "stylus": "^0.54.5",
     "typescript": "^4.1.2",
+    "victory": "^35.4.6",
     "whatwg-fetch": "^3.5.0",
+    "with-query": "^1.1.2",
     "xlsx": "0.16.1"
   },
   "keywords": [

push-site.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 TODAY=$(date +"%Y-%m-%d")
-BUCKET="open.fda.gov-$TODAY"
+BUCKET="open.fda.gov-demo-data-explorer"
 
 echo "Pushing to: ${BUCKET}"
 

src/components/ApiStatus.jsx

@@ -12,6 +12,7 @@ type tPROPS = {
 
 const catMap: Object = Object.freeze({
   'animalandveterinarydrugevent': 'Animal & Veterinary › Adverse Events',
+  'animalandveterinarylabel': 'Animal & Veterinary › Labeling',
   'foodenforcement': 'Foods › Enforcement Reports',
   'foodevent': 'Foods › Adverse Events',
   'drugevent': 'Drugs › Adverse Events',
@@ -36,6 +37,7 @@ const catMap: Object = Object.freeze({
 
 const endpointLinkMap: Object = Object.freeze({
   'animalandveterinarydrugevent': 'animalandveterinary/event',
+  'animalandveterinarylabel': 'animalandveterinary/label',
   'foodevent': 'food/event',
   'foodenforcement': 'food/enforcement',
   'drugevent': 'drug/event',

src/components/ApiUsage.jsx

@@ -305,6 +305,7 @@ const ApiUsage = (props:tPROPS) => {
                     <td colSpan="2"><strong>Animal & Veterinary</strong></td>
                   </tr>
                   <tr><td>Adverse Event Reports</td><td>{this.docCount('animalandveterinarydrugevent')}</td></tr>
+                  <tr><td>Labeling</td><td>{this.docCount('animalandveterinarylabel')}</td></tr>
                   <tr className="bg-primary-darkest clr-white">
                     <td colSpan="2"><strong>Drugs</strong></td>
                   </tr>
@@ -354,6 +355,7 @@ const ApiUsage = (props:tPROPS) => {
                       <td colSpan="2"><strong>Animal & Veterinary</strong></td>
                     </tr>
                     <tr><td>Adverse Event Reports</td><td>{this.downloadCount('animalandveterinarydrugevent')}</td></tr>
+                    <tr><td>Labeling</td><td>{this.downloadCount('animalandveterinarylabel')}</td></tr>
                     <tr className="bg-primary-darkest clr-white">
                       <td colSpan="2"><strong>Drugs</strong></td>
                     </tr>

src/components/AutoComplete.jsx

@@ -0,0 +1,113 @@
+import Autosuggest from 'react-autosuggest'
+import withQuery from 'with-query'
+import 'whatwg-fetch'
+import React from 'react'
+
+class AutoCompleteComponent extends React.Component {
+
+    constructor (props: Object) {
+        super(props)
+
+        this.state = {
+            value: '',
+            suggestions: [],
+            url: this.props.parent.state.dataset.url,
+            endpoint: this.props.parent.state.dataset.endpoint
+        }
+        this.getSuggestions = this.getSuggestions.bind(this)
+        this.onChange = this.onChange.bind(this)
+        this.onSuggestionsClearRequested = this.onSuggestionsClearRequested.bind(this)
+        this.onSuggestionsFetchRequested = this.onSuggestionsFetchRequested.bind(this)
+        this.onSuggestionSelected = this.onSuggestionSelected.bind(this)
+    }
+
+    // https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions#Using_Special_Characters
+    escapeRegexCharacters(str) {
+        return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    }
+
+    renderSuggestion(suggestion) {
+        return (
+            <span>{suggestion}</span>
+        );
+    }
+
+    getSuggestions(value) {
+        return fetch(
+            withQuery(`${this.state.url}/${this.state.endpoint}`,{
+                searchField: this.props.option.field,
+                searchText: value,
+                searchType: 'autocomplete',
+                limit: this.props.option.limit
+            })
+        )
+        .then(res => res.json())
+        .then((json) => json.results)
+        .catch((err) => [])
+    }
+
+    onSuggestionsFetchRequested(value){
+        var that = this;
+        const escapedValue = this.escapeRegexCharacters(value.value.trim())
+        this.getSuggestions(escapedValue).then(result => {
+            if(!result){
+                return
+            }
+            that.setState({
+                suggestions: result,
+                value: escapedValue
+            })
+            console.log("suggestions are:", result);
+        })
+    }
+    onSuggestionSelected(event, { suggestion, suggestionValue, suggestionIndex, sectionIndex, method }){
+        if(this.props.onChange){
+            this.props.onChange(suggestion, {
+                field: this.props.option.field,
+                idx: this.props.option.idx
+            })
+        }
+        console.log(event, suggestion)
+    }
+
+    getSuggestionValue(suggestion) {
+        return suggestion
+    }
+
+    onChange (event, { newValue, method }) {
+        if(!newValue.length){
+            this.setState({
+                suggestions: [],
+                value: newValue
+            })
+        }
+    }
+
+    onSuggestionsClearRequested (){
+        this.setState({
+            suggestions: []
+        })
+    }
+
+    render() {
+        return (
+            <Autosuggest
+                suggestions={this.state.suggestions}
+                onSuggestionSelected={this.onSuggestionSelected}
+                onSuggestionsFetchRequested={this.onSuggestionsFetchRequested}
+                onSuggestionsClearRequested={this.onSuggestionsClearRequested}
+                getSuggestionValue={this.getSuggestionValue}
+                renderSuggestion={this.renderSuggestion}
+                inputProps={{
+                    placeholder: this.props.option.placeholder,
+                    value: this.state.value,
+                    onChange: this.onChange
+                }}
+            />
+        );
+    }
+}
+
+
+AutoCompleteComponent.displayName = 'component/AutoCompleteComponent'
+export default AutoCompleteComponent

src/components/DataRetrieval.jsx

@@ -0,0 +1,174 @@
+/* @flow */
+
+
+import withQuery from 'with-query'
+import {default as $} from 'jquery'
+import Moment from 'moment'
+import {API_LINK} from "../constants/api";
+import update from "immutability-helper/index";
+
+class DataRetrievalService {
+
+  constructor (url, endpoint) {
+    this.url = url
+    this.endpoint = endpoint
+
+    this.convertFiltersToJson = this.convertFiltersToJson.bind(this)
+    this.getTopValues = this.getTopValues.bind(this)
+    this.getTopValuesByIterating = this.getTopValuesByIterating.bind(this)
+    this.getData = this.getData.bind(this)
+    this.getTotal = this.getTotal.bind(this)
+  }
+
+  handleErrors(response) {
+    if (!response.ok) throw Error(response.statusText);
+    return response;
+  }
+
+  addValue (filters, field, value) {
+    return filters.map(filter => {
+      if (filter.field === field) {
+        return update(filter, {value: {$set: value}})
+      } else {
+        return filter
+      }
+    })
+  }
+
+  convertFiltersToJson(filters, options){
+    const formattedFilters = filters.filter(filter => filter.value.length).map((filter,idx) => {
+      var value = {
+        "query-type": filter.query_type,
+        "key": filter.field,
+        "value": filter.value,
+        "type": filter.type
+      }
+      if (value["query-type"] == "range" && value["type"] == "numeric_range") {
+          value.value = {
+              "gte": value.value[0],
+              "lte": value.value[1]
+          }
+      } else if (value["query-type"] == "range") {
+        value.value = {
+          "gte": Moment(value.value[0]).format('YYYYMMDD'),
+          "lte": Moment(value.value[1]).format('YYYYMMDD')
+        }
+      } else if (value["query-type"] == "exists"){
+        value.value = filter.value[0].toString()
+      }
+      return value
+    })
+
+    if(options.drugtype){
+      formattedFilters.push({
+        "query-type": "term",
+        "key": "@drugtype",
+        "value": [
+          options.drugtype
+        ]
+      })
+    }
+
+    if(options.groupingField) {
+      return {
+        "data": {
+          "queryJSON": {
+            "size": 5000,
+            "searchType": "aggregation",
+            "groupingField": options.groupingField,
+            "filters": formattedFilters
+          }
+        }
+      }
+    }
+
+    return {
+      "data": {
+        "queryJSON": {
+          "size": 5000,
+          "searchType": options.searchType,
+          "filters": formattedFilters
+        }
+      }
+    }
+  }
+
+  getTopValues(field){
+    return fetch(
+      withQuery(`${this.url}/${this.endpoint}`,{
+        count: field
+      },{
+        mode: 'cors'
+      })
+    ).then(this.handleErrors)
+    .then(res => res.json())
+    .then((json) => {
+      const res = json.results.map(obj => {
+        return {
+          label: obj.term,
+          value: obj.term
+        }
+      });
+      return res.sort( (a,b) => (a.value > b.value) - (a.value < b.value))
+    }).catch((err) => console.log("err: ", err))
+  }
+
+
+  getTopValuesByIterating(fields, removeChars){
+
+    const limit = 200
+    let urls = []
+
+    for (var i = 0; i < (limit/100); i++) {
+      urls.push(
+        withQuery(`${this.url}/${this.endpoint}`, {
+          limit: 100,
+          skip: i*100
+        })
+      )
+    }
+
+    const itemPromises = urls.map($.getJSON)
+    return Promise.all(itemPromises)
+      .then((results) => {
+        const response = []
+        results.forEach(r => {
+          response.push(...r.results)
+        })
+        return response
+      })
+  }
+
+
+  getData(params, options){
+    const data = this.convertFiltersToJson(params, options)
+    console.log(JSON.stringify(data, null, 4));
+    return fetch(`${this.url}/${this.endpoint}`, {
+      body: JSON.stringify(data),
+      headers: {
+        "Content-Type": "application/json"
+      },
+      method: 'POST',
+      mode: 'cors'
+    }).then(this.handleErrors)
+    .then(res => res.json())
+    .then(res => {
+      return res
+    })
+    .catch((err) => {})
+  }
+
+
+  getTotal() {
+    return fetch(`${this.url}/${this.endpoint}`)
+    .then(this.handleErrors)
+    .then(res => res.json())
+    .then(res => {
+      console.log(res)
+      return res
+    })
+    .catch((err) => {})
+  }
+}
+
+export default DataRetrievalService