diff --git a/Cargo.lock b/Cargo.lock index d6d2132f..b7fedb20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1059,8 +1059,8 @@ dependencies = [ "include_dir", "infisearch_common", "infisearch_lang_ascii", + "infisearch_lang_ascii_stemmer", "infisearch_lang_chinese", - "infisearch_lang_latin", "lazy_static", "log", "log4rs", @@ -1101,23 +1101,23 @@ dependencies = [ ] [[package]] -name = "infisearch_lang_chinese" +name = "infisearch_lang_ascii_stemmer" version = "0.9.1" dependencies = [ "infisearch_common", "infisearch_lang_ascii", - "lazy_static", - "regex", + "rust-stemmers", "smartstring", ] [[package]] -name = "infisearch_lang_latin" +name = "infisearch_lang_chinese" version = "0.9.1" dependencies = [ "infisearch_common", "infisearch_lang_ascii", - "rust-stemmers", + "lazy_static", + "regex", "smartstring", ] @@ -1129,8 +1129,8 @@ dependencies = [ "byteorder", "infisearch_common", "infisearch_lang_ascii", + "infisearch_lang_ascii_stemmer", "infisearch_lang_chinese", - "infisearch_lang_latin", "js-sys", "miniserde", "pretty_assertions", diff --git a/Makefile b/Makefile index 81720332..2dd63d8b 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ releaseAsciiLanguage: # These 2 are separate as the prior needs to be published first preReleaseOtherLanguages: - cd packages/infisearch_languages/infisearch_lang_latin &&\ + cd packages/infisearch_languages/infisearch_lang_ascii_stemmer &&\ cargo package &&\ cargo package --list cd packages/infisearch_languages/infisearch_lang_chinese &&\ @@ -39,7 +39,7 @@ preReleaseOtherLanguages: cargo package --list releaseOtherLanguages: - cd packages/infisearch_languages/infisearch_lang_latin &&\ + cd packages/infisearch_languages/infisearch_lang_ascii_stemmer &&\ cargo publish cd packages/infisearch_languages/infisearch_lang_chinese &&\ cargo publish diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 50ac5390..16fcfd62 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -9,11 +9,11 @@ - [CSP](./csp.md) - [Search UI](./search_configuration.md) - [Styling](./search_configuration_styling.md) -- [Language](./language.md) - [Indexer](./indexer_configuration.md) - [Fields](./indexer/fields.md) - [Files](./indexer/files.md) - [Miscellaneous](./indexer/misc.md) +- [Language](./language.md) - [Search API](./search_api.md) - [Query Syntax](./search_syntax.md) - [Larger Collections](./larger_collections.md) diff --git a/docs/src/language.md b/docs/src/language.md index eee31aad..f289a25e 100644 --- a/docs/src/language.md +++ b/docs/src/language.md @@ -2,13 +2,19 @@ There are 3 language modules available. To configure these, you will need to serve the appropriate [language bundle](./getting_started.md#hosting-the-files) in your HTML (or edit the CDN link accordingly), and edit the indexer configuration file. -## Ascii Tokenizer +```json +{ + "lang_config": { + // ... options go here ... + } +} +``` -#### CDN link +## Ascii Tokenizer -The default tokenizer splits on sentences, then whitespaces to obtain tokens. +The default tokenizer should work for any language that relies on ASCII characters, or their inflections (e.g. "รก"). -An [asciiFoldingFilter](https://github.com/tantivy-search/tantivy/blob/main/src/tokenizer/ascii_folding_filter.rs) is then applied to these tokens, followed by punctuation and non-word-character boundary removal. +The text is first split into on sentences, then whitespaces to obtain tokens. An [asciiFoldingFilter](https://github.com/tantivy-search/tantivy/blob/main/src/tokenizer/ascii_folding_filter.rs) is then applied to normalize diacritics, followed by punctuation and non-word-character boundary removal. ```json { @@ -34,13 +40,13 @@ An [asciiFoldingFilter](https://github.com/tantivy-search/tantivy/blob/main/src/ ``` -## Latin Tokenizer +## Ascii Tokenizer with Stemmer This is essentially the same as the ascii tokenizer, but adds a `stemmer` option. ```json { - "lang": "latin", + "lang": "ascii_stemmer", "options": { // ---------------------------------- // Ascii Tokenizer options also apply @@ -60,7 +66,7 @@ If you do not need stemming, use the `ascii` tokenizer, which has a smaller wasm **CDN Link** ```html - + ``` ## Chinese Tokenizer diff --git a/e2e/e2e.test.js b/e2e/e2e.test.js index de647d93..af0b67e0 100644 --- a/e2e/e2e.test.js +++ b/e2e/e2e.test.js @@ -56,10 +56,16 @@ const testSuite = async (configFile, with_positions, with_filters) => { await reloadPage(lang); + const isAsciiStemmer = lang === 'ascii_stemmer'; + // ------------------------------------------------------ // Various basic tests on docid=0 await typeText('+npm +run +dev +installmdbook'); - await assertSingle('use the npm run dev script'); + await assertSingle( + isAsciiStemmer + ? 'then run npm run devServer1 to' + : 'use the npm run dev script', + ); if (with_positions) { await typeText('+"npm run dev" +(installmdbook 8080)'); @@ -399,8 +405,12 @@ const testSuite = async (configFile, with_positions, with_filters) => { 'detecting such terms', 'the change detection', 'detectedd as per the earlier section', - 'mobile device detection', ]; + if (!isAsciiStemmer) { + // The stemmer version generates 'detect' and 'detectedd', + // weighting the other sub result more heavily and discarding this one. + expectedPrefixResults.push('mobile device detection'); + } await typeText('detec'); await assertMultiple(expectedPrefixResults, 2); @@ -427,7 +437,11 @@ const testSuite = async (configFile, with_positions, with_filters) => { await assertSingle('lorem ipsum is simply dummy text'); await typePhraseOrAnd('test many json 2', with_positions); - await assertSingle('test many json 2'); + if (isAsciiStemmer) { + await assertMultiple(['test many json 2', 'estimate from testing'], 2); + } else { + await assertSingle('test many json 2'); + } // ------------------------------------------------------ // ------------------------------------------------------ @@ -454,8 +468,14 @@ const testSuite = async (configFile, with_positions, with_filters) => { runFullIndex(configFile); // 1, to be deleted later + + // For AND queries, make it slightly more specific + const test404Query = 'This URL is invaldi' + (with_positions ? '' : ' navigation'); + const testContributionsQuery = 'Contributions of any form' + (with_positions ? '' : ' development'); + const testContributionsUpdatedQuery = 'Contributions of all forms' + (with_positions ? '' : ' atquejxusd'); + await reloadPage(lang); - await typePhraseOrAnd('This URL is invaldi', with_positions); + await typePhraseOrAnd(test404Query, with_positions); await waitNoResults(); fs.copyFileSync( @@ -465,11 +485,11 @@ const testSuite = async (configFile, with_positions, with_filters) => { runIncrementalIndex(configFile); await reloadPage(lang); - await typePhraseOrAnd('This URL is invaldi', with_positions); + await typePhraseOrAnd(test404Query, with_positions); await assertSingle('this url is invalid'); // 2, to be updated later - await typePhraseOrAnd('Contributions of any form', with_positions); + await typePhraseOrAnd(testContributionsQuery, with_positions); await waitNoResults(); const contributingHtmlOutputPath = path.join(__dirname, 'input/contributing.html'); @@ -480,7 +500,7 @@ const testSuite = async (configFile, with_positions, with_filters) => { runIncrementalIndex(configFile); await reloadPage(lang); - await typePhraseOrAnd('Contributions of any form', with_positions); + await typePhraseOrAnd(testContributionsQuery, with_positions); await assertSingle('contributions of any form'); // ------------------------------------------------------ @@ -494,7 +514,7 @@ const testSuite = async (configFile, with_positions, with_filters) => { runIncrementalIndex(configFile); await reloadPage(lang); - await typePhraseOrAnd('This URL is invaldi', with_positions); + await typePhraseOrAnd(test404Query, with_positions); await waitNoResults(); expectNumDeletedDocs(1); @@ -504,7 +524,7 @@ const testSuite = async (configFile, with_positions, with_filters) => { // ------------------------------------------------------ // Test incremental indexing update - await typePhraseOrAnd('Contributions of all forms', with_positions); + await typePhraseOrAnd(testContributionsUpdatedQuery, with_positions); await waitNoResults(); let contributingHtml = fs.readFileSync(contributingHtmlOutputPath, 'utf-8'); @@ -515,10 +535,10 @@ const testSuite = async (configFile, with_positions, with_filters) => { runIncrementalIndex(configFile); await reloadPage(lang); - await typePhraseOrAnd('Contributions of any form', with_positions); + await typePhraseOrAnd(testContributionsQuery, with_positions); await waitNoResults(); - await typePhraseOrAnd('Contributions of all forms', with_positions); + await typePhraseOrAnd(testContributionsUpdatedQuery, with_positions); await assertSingle('contributions of all forms'); await typeText('atquejxusd '); @@ -532,10 +552,10 @@ const testSuite = async (configFile, with_positions, with_filters) => { runIncrementalIndex(configFile); await reloadPage(lang); - await typePhraseOrAnd('Contributions of any form', with_positions); + await typePhraseOrAnd(testContributionsQuery, with_positions); await waitNoResults(); - await typePhraseOrAnd('Contributions of all forms', with_positions); + await typePhraseOrAnd(testContributionsUpdatedQuery, with_positions); await waitNoResults(); await typeText('atquejxusd'); @@ -678,7 +698,7 @@ const mainTest = async () => { outputConfig = readOutputConfig(); expect(outputConfig.indexingConfig.plNamesToCache).toHaveLength(0); - // Latin tokenizer + // Ascii with stemmer tokenizer // No positions cleanup(); console.log('Starting infi_search_4 tests'); diff --git a/e2e/input/infi_search_4.json b/e2e/input/infi_search_4.json index 999ed04a..5ffbbb03 100644 --- a/e2e/input/infi_search_4.json +++ b/e2e/input/infi_search_4.json @@ -6,7 +6,7 @@ "fields": {} }, "lang_config": { - "lang": "ascii", + "lang": "ascii_stemmer", "options": { "ignore_stop_words": false } diff --git a/packages/infisearch/Cargo.toml b/packages/infisearch/Cargo.toml index 7e65a580..3abe9361 100644 --- a/packages/infisearch/Cargo.toml +++ b/packages/infisearch/Cargo.toml @@ -35,7 +35,7 @@ log = { version = "0.4", features = ["max_level_info", "release_max_level_info"] log4rs = "1.0" infisearch_common = { path = "../infisearch_common", version="=0.9.1", features = ["indexer"] } infisearch_lang_ascii = { path = "../infisearch_languages/infisearch_lang_ascii", version="=0.9.1", features = ["indexer"] } -infisearch_lang_latin = { path = "../infisearch_languages/infisearch_lang_latin", version="=0.9.1", features = ["indexer"] } +infisearch_lang_ascii_stemmer = { path = "../infisearch_languages/infisearch_lang_ascii_stemmer", version="=0.9.1", features = ["indexer"] } infisearch_lang_chinese = { path = "../infisearch_languages/infisearch_lang_chinese", version="=0.9.1", features = ["indexer"] } num_cpus = "1" path-absolutize = { version = "3.0", features = ["lazy_static_cache"] } diff --git a/packages/infisearch/src/indexer.rs b/packages/infisearch/src/indexer.rs index 4ab6a0f5..d42aeedc 100644 --- a/packages/infisearch/src/indexer.rs +++ b/packages/infisearch/src/indexer.rs @@ -15,7 +15,7 @@ use infisearch_common::language::InfiLanguageConfig; use infisearch_common::METADATA_FILE; use infisearch_common::tokenize::IndexerTokenizer; use infisearch_lang_ascii::ascii; -use infisearch_lang_latin::latin; +use infisearch_lang_ascii_stemmer::ascii_stemmer; use infisearch_lang_chinese::chinese; use crate::dictionary_writer::DictWriter; @@ -247,7 +247,7 @@ impl Indexer { fn resolve_tokenizer(lang_config: &InfiLanguageConfig) -> Arc { match lang_config.lang.as_str() { "ascii" => Arc::new(ascii::new_with_options(lang_config)), - "latin" => Arc::new(latin::new_with_options(lang_config)), + "ascii_stemmer" => Arc::new(ascii_stemmer::new_with_options(lang_config)), "chinese" => Arc::new(chinese::new_with_options(lang_config)), _ => panic!("Unsupported language {}", lang_config.lang), } diff --git a/packages/infisearch_languages/infisearch_lang_latin/Cargo.toml b/packages/infisearch_languages/infisearch_lang_ascii_stemmer/Cargo.toml similarity index 93% rename from packages/infisearch_languages/infisearch_lang_latin/Cargo.toml rename to packages/infisearch_languages/infisearch_lang_ascii_stemmer/Cargo.toml index c4e65760..2a290483 100644 --- a/packages/infisearch_languages/infisearch_lang_latin/Cargo.toml +++ b/packages/infisearch_languages/infisearch_lang_ascii_stemmer/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "infisearch_lang_latin" +name = "infisearch_lang_ascii_stemmer" version = "0.9.1" authors = ["Ze Yu "] edition = "2018" diff --git a/packages/infisearch_languages/infisearch_lang_latin/src/latin.rs b/packages/infisearch_languages/infisearch_lang_ascii_stemmer/src/ascii_stemmer.rs similarity index 100% rename from packages/infisearch_languages/infisearch_lang_latin/src/latin.rs rename to packages/infisearch_languages/infisearch_lang_ascii_stemmer/src/ascii_stemmer.rs diff --git a/packages/infisearch_languages/infisearch_lang_ascii_stemmer/src/lib.rs b/packages/infisearch_languages/infisearch_lang_ascii_stemmer/src/lib.rs new file mode 100644 index 00000000..d25288f1 --- /dev/null +++ b/packages/infisearch_languages/infisearch_lang_ascii_stemmer/src/lib.rs @@ -0,0 +1 @@ +pub mod ascii_stemmer; diff --git a/packages/infisearch_languages/infisearch_lang_latin/src/lib.rs b/packages/infisearch_languages/infisearch_lang_latin/src/lib.rs deleted file mode 100644 index b5339241..00000000 --- a/packages/infisearch_languages/infisearch_lang_latin/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod latin; diff --git a/packages/infisearch_search/Cargo.toml b/packages/infisearch_search/Cargo.toml index b5274aae..85eb496b 100644 --- a/packages/infisearch_search/Cargo.toml +++ b/packages/infisearch_search/Cargo.toml @@ -10,7 +10,7 @@ crate-type = ["cdylib"] [features] default = ["lang_ascii"] lang_ascii = ["infisearch_lang_ascii"] -lang_latin = ["infisearch_lang_latin"] +lang_ascii_stemmer = ["infisearch_lang_ascii_stemmer"] lang_chinese = ["infisearch_lang_chinese"] perf = ["web-sys"] @@ -20,7 +20,7 @@ byteorder = "1" js-sys = "0.3.51" infisearch_common = { path = "../infisearch_common", features = [] } infisearch_lang_ascii = { path = "../infisearch_languages/infisearch_lang_ascii", optional = true, features = [] } -infisearch_lang_latin = { path = "../infisearch_languages/infisearch_lang_latin", optional = true, features = [] } +infisearch_lang_ascii_stemmer = { path = "../infisearch_languages/infisearch_lang_ascii_stemmer", optional = true, features = [] } infisearch_lang_chinese = { path = "../infisearch_languages/infisearch_lang_chinese", optional = true, features = [] } smartstring = "0.2.7" wasm-bindgen = { version = "0.2" } diff --git a/packages/infisearch_search/pkg/lang_latin/.gitignore b/packages/infisearch_search/pkg/lang_ascii_stemmer/.gitignore similarity index 100% rename from packages/infisearch_search/pkg/lang_latin/.gitignore rename to packages/infisearch_search/pkg/lang_ascii_stemmer/.gitignore diff --git a/packages/infisearch_search/pkg/lang_latin/package.json b/packages/infisearch_search/pkg/lang_ascii_stemmer/package.json similarity index 85% rename from packages/infisearch_search/pkg/lang_latin/package.json rename to packages/infisearch_search/pkg/lang_ascii_stemmer/package.json index 0a4bc818..62319631 100644 --- a/packages/infisearch_search/pkg/lang_latin/package.json +++ b/packages/infisearch_search/pkg/lang_ascii_stemmer/package.json @@ -1,5 +1,5 @@ { - "name": "@infisearch/lang-latin", + "name": "@infisearch/lang-ascii-stemmer", "collaborators": [ "Ze Yu " ], diff --git a/packages/infisearch_search/src/searcher.rs b/packages/infisearch_search/src/searcher.rs index 89cba946..33f80d42 100644 --- a/packages/infisearch_search/src/searcher.rs +++ b/packages/infisearch_search/src/searcher.rs @@ -23,8 +23,8 @@ use crate::utils; #[cfg(feature = "lang_ascii")] use infisearch_lang_ascii::ascii; -#[cfg(feature = "lang_latin")] -use infisearch_lang_latin::latin; +#[cfg(feature = "lang_ascii_stemmer")] +use infisearch_lang_ascii_stemmer::ascii_stemmer; #[cfg(feature = "lang_chinese")] use infisearch_lang_chinese::chinese; @@ -80,9 +80,9 @@ fn get_tokenizer(lang_config: &InfiLanguageConfig) -> Box { Box::new(ascii::new_with_options(lang_config)) } -#[cfg(feature = "lang_latin")] +#[cfg(feature = "lang_ascii_stemmer")] fn get_tokenizer(lang_config: &InfiLanguageConfig) -> Box { - Box::new(latin::new_with_options(lang_config)) + Box::new(ascii_stemmer::new_with_options(lang_config)) } #[cfg(feature = "lang_chinese")] @@ -441,7 +441,7 @@ pub mod test { with_positions: true, }, lang_config: InfiLanguageConfig { - lang: "latin".to_owned(), + lang: "ascii_stemmer".to_owned(), options: InfiLanguageConfigOpts::default(), }, field_infos, diff --git a/packages/infisearch_search/src/searcher/query_parser.rs b/packages/infisearch_search/src/searcher/query_parser.rs index 97f7d6bc..40ee95db 100644 --- a/packages/infisearch_search/src/searcher/query_parser.rs +++ b/packages/infisearch_search/src/searcher/query_parser.rs @@ -701,7 +701,7 @@ pub mod test { pub fn parse_wo_pos(query: &str) -> Vec { let tokenizer = ascii::new_with_options(&InfiLanguageConfig { - lang: "latin".to_owned(), + lang: "ascii_stemmer".to_owned(), options: InfiLanguageConfigOpts::default(), }); diff --git a/packages/mdbook-infisearch/src/main.rs b/packages/mdbook-infisearch/src/main.rs index 42ef6635..dd12e90a 100644 --- a/packages/mdbook-infisearch/src/main.rs +++ b/packages/mdbook-infisearch/src/main.rs @@ -199,7 +199,8 @@ const base_url = '{}'; const mode = {}; {} ", - base_url, lang, + base_url, + lang.replace("_", "-"), base_url, base_url, mode, diff --git a/packages/search-ui/public/template.html b/packages/search-ui/public/template.html index a2573fe1..a720332f 100644 --- a/packages/search-ui/public/template.html +++ b/packages/search-ui/public/template.html @@ -100,10 +100,10 @@

InfiSearch Dev Site

function toggleLang() { if (window.location.pathname.includes('chinese-lang')) { window.location.href = window.location.href.replace(/chinese-lang/, 'ascii-lang'); - } else if (window.location.pathname.includes('latin-lang')) { - window.location.href = window.location.href.replace(/latin-lang/, 'chinese-lang'); + } else if (window.location.pathname.includes('ascii_stemmer-lang')) { + window.location.href = window.location.href.replace(/ascii_stemmer-lang/, 'chinese-lang'); } else { - window.location.href = window.location.href.replace(/ascii-lang/, 'latin-lang'); + window.location.href = window.location.href.replace(/ascii-lang/, 'ascii_stemmer-lang'); } } diff --git a/packages/search-ui/src/entries/latin.ts b/packages/search-ui/src/entries/ascii-stemmer.ts similarity index 86% rename from packages/search-ui/src/entries/latin.ts rename to packages/search-ui/src/entries/ascii-stemmer.ts index 7a6b91c1..039dbe44 100644 --- a/packages/search-ui/src/entries/latin.ts +++ b/packages/search-ui/src/entries/ascii-stemmer.ts @@ -1,5 +1,5 @@ // eslint-disable-next-line @typescript-eslint/no-unused-vars -import { Searcher } from '@infisearch/search-lib/lib/results/Searcher/Searcher-latin'; +import { Searcher } from '@infisearch/search-lib/lib/results/Searcher/Searcher-ascii-stemmer'; import init from '../search'; export default { diff --git a/packages/search/lib/results/Query.ts b/packages/search/lib/results/Query.ts index 9f6cb0e0..5fd76bd5 100644 --- a/packages/search/lib/results/Query.ts +++ b/packages/search/lib/results/Query.ts @@ -41,7 +41,7 @@ export function getRegexes(queryParts: QueryPart[], config: InfiConfig) { .sort((a, b) => b.length - a.length) .join('|'); - if (config.langConfig.lang === 'latin') { + if (config.langConfig.lang === 'ascii_stemmer') { const nonEndBoundariedRegex = new RegExp(`(^|\\W|_)(${innerTermsJoined})(\\w*?)(?=\\W|$)`, 'gi'); termRegexes.push(nonEndBoundariedRegex); } else { diff --git a/packages/search/lib/results/Searcher/Searcher-ascii-stemmer.ts b/packages/search/lib/results/Searcher/Searcher-ascii-stemmer.ts new file mode 100644 index 00000000..9facc68d --- /dev/null +++ b/packages/search/lib/results/Searcher/Searcher-ascii-stemmer.ts @@ -0,0 +1,12 @@ +// @ts-ignore +import workerScript from '../../../worker-dist/search-worker-ascii-stemmer.bundle?raw'; +import Searcher from '../Searcher'; +import { workerScript as SearcherScript } from '../Searcher'; +import Query from '../Query'; + +SearcherScript.s = workerScript; + +export { + Searcher, + Query, +}; diff --git a/packages/search/lib/results/Searcher/Searcher-latin.ts b/packages/search/lib/results/Searcher/Searcher-latin.ts deleted file mode 100644 index 997d0130..00000000 --- a/packages/search/lib/results/Searcher/Searcher-latin.ts +++ /dev/null @@ -1,12 +0,0 @@ -// @ts-ignore -import workerScript from '../../../worker-dist/search-worker-latin.bundle?raw'; -import Searcher from '../../results/Searcher'; -import { workerScript as SearcherScript } from '../../results/Searcher'; -import Query from '../../results/Query'; - -SearcherScript.s = workerScript; - -export { - Searcher, - Query, -}; diff --git a/packages/search/lib/worker/languages/worker-latin.ts b/packages/search/lib/worker/languages/worker-ascii-stemmer.ts similarity index 87% rename from packages/search/lib/worker/languages/worker-latin.ts rename to packages/search/lib/worker/languages/worker-ascii-stemmer.ts index b11afac9..9ef03a94 100644 --- a/packages/search/lib/worker/languages/worker-latin.ts +++ b/packages/search/lib/worker/languages/worker-ascii-stemmer.ts @@ -5,5 +5,5 @@ import setupWithWasmModule from '../worker'; setupWithWasmModule(import( /* webpackMode: "eager" */ /* webpackExports: ["get_new_searcher", "get_query"] */ - '@infisearch/lang-latin' + '@infisearch/lang-ascii-stemmer' )); diff --git a/packages/search/package.json b/packages/search/package.json index 319ba991..423f873b 100644 --- a/packages/search/package.json +++ b/packages/search/package.json @@ -31,6 +31,6 @@ "devDependencies": { "@infisearch/lang-ascii": "^0.9.1", "@infisearch/lang-chinese": "^0.9.1", - "@infisearch/lang-latin": "^0.9.1" + "@infisearch/lang-ascii-stemmer": "^0.9.1" } } diff --git a/webpack.common.js b/webpack.common.js index f3ac0e96..d531d82f 100644 --- a/webpack.common.js +++ b/webpack.common.js @@ -21,7 +21,7 @@ function getLangConfig(lang) { module.exports = { entry: { 'search-ui-ascii': getLangConfig('ascii'), - 'search-ui-latin': getLangConfig('latin'), + 'search-ui-ascii_stemmer': getLangConfig('ascii-stemmer'), 'search-ui-chinese': getLangConfig('chinese'), 'search-ui-basic': { import: path.resolve(__dirname, 'packages/search-ui/src/styles/basic.css'), diff --git a/webpack.dev.js b/webpack.dev.js index 43e51081..caa63651 100644 --- a/webpack.dev.js +++ b/webpack.dev.js @@ -72,7 +72,7 @@ module.exports = (env) => merge(common, { }; const themes = ['basic', 'light', 'dark']; - const languages = ['ascii', 'latin', 'chinese']; + const languages = ['ascii', 'ascii_stemmer', 'chinese']; const plugins = []; for (const theme of themes) { diff --git a/webpack.worker.js b/webpack.worker.js index 0c2ec32c..a8241b26 100644 --- a/webpack.worker.js +++ b/webpack.worker.js @@ -18,7 +18,7 @@ module.exports = (env) => { mode: 'development', entry: { 'search-worker-ascii': getWorkerLangConfig('ascii'), - 'search-worker-latin': getWorkerLangConfig('latin'), + 'search-worker-ascii_stemmer': getWorkerLangConfig('ascii-stemmer'), 'search-worker-chinese': getWorkerLangConfig('chinese'), }, output: { @@ -51,9 +51,9 @@ module.exports = (env) => { }), new WasmPackPlugin({ crateDirectory: path.resolve(__dirname, './packages/infisearch_search'), - extraArgs: '-- --no-default-features --features lang_latin' + perfOption + extraArgs: '-- --no-default-features --features lang_ascii_stemmer' + perfOption + ' -Z build-std=std,panic_abort -Z build-std-features=panic_immediate_abort', - outDir: path.resolve(__dirname, './packages/infisearch_search/pkg/lang_latin'), + outDir: path.resolve(__dirname, './packages/infisearch_search/pkg/lang_ascii_stemmer'), ...perfMode, }), new WasmPackPlugin({