Skip to content

Commit

Permalink
update ingest api calls
Browse files Browse the repository at this point in the history
  • Loading branch information
@rickjohnson
rickjohnson committed Feb 21, 2024
1 parent cf2903f commit 4d7d7fd
Show file tree
Hide file tree
Showing 3 changed files with 79 additions and 21 deletions.
2 changes: 1 addition & 1 deletion client/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
"graphql-tag": "^2.10.3",
"lodash": "4.17.21",
"mathjax": "3",
"meilisearch": "^0.11.0",
"meilisearch": "^0.28.0",
"moment": "^2.24.0",
"node-fetch": "^2.6.1",
"p-map": "^4.0.0",
Expand Down
49 changes: 37 additions & 12 deletions client/yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2403,13 +2403,6 @@ axios@^0.18.1:
follow-redirects "1.5.10"
is-buffer "^2.0.2"

axios@^0.19.2:
version "0.19.2"
resolved "https://registry.yarnpkg.com/axios/-/axios-0.19.2.tgz#3ea36c5d8818d0d5f8a8a97a6d36b86cdc00cb27"
integrity sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==
dependencies:
follow-redirects "1.5.10"

babel-eslint@^10.0.1:
version "10.0.3"
resolved "https://registry.yarnpkg.com/babel-eslint/-/babel-eslint-10.0.3.tgz#81a2c669be0f205e19462fed2482d33e4687a88a"
Expand Down Expand Up @@ -3628,6 +3621,13 @@ cross-fetch@^3.1.4:
dependencies:
node-fetch "2.6.1"

cross-fetch@^3.1.5:
version "3.1.8"
resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.8.tgz#0327eba65fd68a7d119f8fb2bf9334a1a7956f82"
integrity sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==
dependencies:
node-fetch "^2.6.12"

[email protected]:
version "7.0.3"
resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
Expand Down Expand Up @@ -7187,12 +7187,12 @@ [email protected]:
resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748"
integrity sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=

meilisearch@^0.11.0:
version "0.11.3"
resolved "https://registry.yarnpkg.com/meilisearch/-/meilisearch-0.11.3.tgz#8b9f3858ca3257c94ca83a70c8697fb02301b059"
integrity sha512-YEUwZj4096otjmUyyQULF1sxQmoy1zfZOmCFO/D8xwtzzIXlcF0B/YjIQTp7FCeNOWCc4JS8emv/hmgEWjxdXA==
meilisearch@^0.28.0:
version "0.28.0"
resolved "https://registry.yarnpkg.com/meilisearch/-/meilisearch-0.28.0.tgz#650f08a56ef0a989a41d13233f7a744cfa7f54df"
integrity sha512-zC50xLyPWBtfAIZiFTSJyYJp/a5bO+dSSigBKUEoShFkuv9+/KC4J3T3ZejNVHXG2DU0ohOMf8TZ3HAHXNd3EA==
dependencies:
axios "^0.19.2"
cross-fetch "^3.1.5"

[email protected], memory-fs@^0.5.0:
version "0.5.0"
Expand Down Expand Up @@ -7650,6 +7650,13 @@ node-fetch@^1.0.1:
encoding "^0.1.11"
is-stream "^1.0.1"

node-fetch@^2.6.12:
version "2.7.0"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
dependencies:
whatwg-url "^5.0.0"

node-forge@^0.10.0:
version "0.10.0"
resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3"
Expand Down Expand Up @@ -10742,6 +10749,11 @@ totalist@^1.0.0:
resolved "https://registry.yarnpkg.com/totalist/-/totalist-1.1.0.tgz#a4d65a3e546517701e3e5c37a47a70ac97fe56df"
integrity sha512-gduQwd1rOdDMGxFG1gEvhV88Oirdo2p+KjoYFU7k2g+i7n6AFFbDQ5kMPUsW0pNbfQsB/cwXvT1i4Bue0s9g5g==

tr46@~0.0.3:
version "0.0.3"
resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==

trim-newlines@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-1.0.0.tgz#5887966bb582a4503a41eb524f7d35011815a613"
Expand Down Expand Up @@ -11306,6 +11318,11 @@ wbuf@^1.1.0, wbuf@^1.7.3:
dependencies:
minimalistic-assert "^1.0.0"

webidl-conversions@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==

[email protected]:
version "4.4.2"
resolved "https://registry.yarnpkg.com/webpack-bundle-analyzer/-/webpack-bundle-analyzer-4.4.2.tgz#39898cf6200178240910d629705f0f3493f7d666"
Expand Down Expand Up @@ -11464,6 +11481,14 @@ whatwg-fetch@>=0.10.0:
resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-3.0.0.tgz#fc804e458cc460009b1a2b966bc8817d2578aefb"
integrity sha512-9GSJUgz1D4MfyKU7KRqwOjXCXTqWdFNvEr7eUBYchQiVc744mqK/MzXPNR2WsPkmkOa4ywfg8C2n8h+13Bey1Q==

whatwg-url@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"
integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==
dependencies:
tr46 "~0.0.3"
webidl-conversions "^3.0.0"

when@~3.6.x:
version "3.6.4"
resolved "https://registry.yarnpkg.com/when/-/when-3.6.4.tgz#473b517ec159e2b85005497a13983f095412e34e"
Expand Down
49 changes: 41 additions & 8 deletions dashboard-search/src/ingest.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,8 @@ function getNormedPersonPublications(reviewPersonPublications) {
async function main() {
console.log(await searchClient.getKeys())
try {
await searchClient.getIndex('publications').deleteIndex()
let index = await searchClient.getIndex('publications')
index.delete()
} catch ( err ) {

}
Expand All @@ -269,7 +270,9 @@ async function main() {
index = await searchClient.getIndex('publications')
}

console.log("-------------index is:-------------")
console.log(index)
console.log("--------------end index is:-----------------")

const thisYear = Number.parseInt(moment().format('YYYY'))
const startYear = thisYear - 4
Expand All @@ -282,7 +285,7 @@ async function main() {
const organizationValues = await getOrganizationValues()
const publicationGraphs: PublicationGraph[] = []

// const organizationValues = ['NDNANO']
// const organizationValues = ['NDNANO','IPH']
for (let i=0; i < organizationValues.length; i++) {
for (let j=0; j < years.length; j++) {
const flatPublications = await loadPublications(organizationValues[i], [years[j]])
Expand Down Expand Up @@ -381,20 +384,50 @@ async function main() {
documents.push(addDoc)
})
})
console.log(`Mapped #: ${documents.length}`)

await index.addDocuments(documents)
// need to add swapping indexes instead of blowing away and ability to roll-back if needed -> keep last
console.log(`Mapped #: ${documents.length}`)
const indexes = await searchClient.getIndexes()
console.log(`${JSON.stringify(indexes['results'])}`)
//index = await searchClient.getIndex('publications')

const status = await searchClient.index('publications').addDocuments(documents)
//indexes['results'].

console.log(`Add Document Task Created: ${JSON.stringify(status)}`)
console.log(`Documents added`)

let status
const { updateId } = await index.updateAttributesForFaceting([
let task
// await searchClient.index('publications').updateFilterableAttributes([
// 'year', 'type', 'journal', 'classifications', 'authors', 'journal_type', 'publisher', 'classificationsTopLevel', 'funder', 'impact_factor_range', 'review_organization_value', 'review_organization_label'
// ])



do {
console.log(".")
await sleep(10)
task = await searchClient.getTask(status.taskUid)
console.log(`${JSON.stringify(task)}`)
} while (task.status == 'processing' || task.status == 'enqueued')
const addedDocuments = await searchClient.index('publications').getDocuments()
console.log(`Added Documents Total: ${JSON.stringify(addedDocuments.results.length)}`)

const facetTask = await searchClient.index('publications').updateFilterableAttributes([
'year', 'type', 'journal', 'classifications', 'authors', 'journal_type', 'publisher', 'classificationsTopLevel', 'funder', 'impact_factor_range', 'review_organization_value', 'review_organization_label'
])

let facetStatus
do {
await sleep(10)
status = await index.getUpdateStatus(updateId)
} while (status.status !== 'processed')
facetStatus = await searchClient.getTask(facetTask.taskUid)
console.log(`Updating facets...${JSON.stringify(facetStatus.status)}`)
} while (facetStatus.status == 'processing' || facetStatus.status == 'enqueued')
console.log(`${JSON.stringify(facetStatus)}`)

const filterableAttributes = await searchClient.index('publications').getFilterableAttributes()
console.log(`Filterable attributes are: ${JSON.stringify(filterableAttributes)}`)

}

main()

0 comments on commit 4d7d7fd

Please sign in to comment.