small app refresh (#49)

- drop py2 support
- update py pkgs
- refactor code for cheroot integration
- update docs
- update yaml settings example
- update virtualenv setup
- fix tests

.gitignore

@@ -7,3 +7,4 @@ data
 local_settings.py
 sherlock-meta.cfg
 local_settings.yml
+.vscode/settings.json

README.md

@@ -11,7 +11,7 @@ Instructions:
 1. Run `sh setup/virtualenv-setup.sh` to setup an isolated environment and download core packages.
 1. Configure settings. The defaults in [`settings.py`](settings.py) provide documentation for each setting.
 	- Copy [`example.local_settings.yml`](example.local_settings.yml) to `local_settings.yml`.
-	- Override/copy any setting from [`settings.py`](settings.py) to `local_settings.yml` (change the values as needed). All YAML keys/options must be lowercase.
+	- Override/copy any setting from [`settings.py`](settings.py) to `local_settings.yml` (change the values as needed). All YAML keys/options **must** be lowercase.
 1. Run `source sherlock_env/bin/activate` to enter the virtual environment.
 1. Run `python main.py --index update` or `--index rebuild` to index the path specified in the settings. Watch indexing output.
 1. Run `python main.py --runserver` to start the web server.
@@ -36,9 +36,9 @@ Includes:
 - End-to-end interface
 	- Indexing and searching text (source code). Built-in support for [whoosh](https://whoosh.readthedocs.io) (fast searching) or [xapian](http://xapian.org/) (much faster searching).
 	    - Easily extend indexing or searching via custom backends.
-	- Front end web app served using [werkzeug](http://werkzeug.pocoo.org/) or [cherrypy](http://www.cherrypy.org/).
+	- Front end web app served using [werkzeug](http://werkzeug.pocoo.org/) or [cheroot](https://cheroot.cherrypy.org).
 	    - `werkzeug` is for development to small traffic.
-	    - `cherrypy` is a high-speed, production ready, thread pooled, generic HTTP server.
+	    - `cheroot` is the high-performance, pure-Python HTTP server used by [CherryPy](https://www.cherrypy.org).
 	- Settings and configuration using [Python](http://python.org).
 
 ### Web Interface
@@ -65,18 +65,18 @@ In [`settings.py`](settings.py):
 
 ## Using other web servers
 
-Text Sherlock has built-in support for [werkzeug](http://werkzeug.pocoo.org/) and [cherrypy](http://www.cherrypy.org/) WSGI compliant servers.
+Text Sherlock has built-in support for [werkzeug](http://werkzeug.pocoo.org/) and [cheroot](https://cheroot.cherrypy.org) WSGI compliant servers.
 
 In [`settings.py`](settings.py):
 
 - Change the `server_type` value to one of the available server types.
     - Possible values:
         - `default`, werkzeug web server (default).
-        - `cherrypy`, production ready web server.
+        - `cheroot`, production ready web server.
 
 ## Core packages
 
-**Requires Python 3.3+**
+**Requires Python 3.5+**
 
 * Whoosh - [whoosh](https://whoosh.readthedocs.io/en/latest/quickstart.html#a-quick-introduction)
 * Flask - [flask](http://flask.pocoo.org)
@@ -91,7 +91,7 @@ In [`settings.py`](settings.py):
 * http://twitter.github.com/bootstrap/examples/container-app.html
 * http://pygments.org/
 * http://docs.peewee-orm.com/
-* http://www.cherrypy.org/
+* https://cheroot.cherrypy.org/
 * http://xapian.org/
 * http://pyyaml.org/wiki/PyYAMLDocumentation
 

core/__init__.py

@@ -11,7 +11,7 @@
     'FULL_INDEX_PATHS'
 ]
 
-from cherrypy import wsgiserver as cherrypy_wsgiserver
+from cheroot.wsgi import Server as WSGIServer, PathInfoDispatcher as WSGIPathInfoDispatcher
 import configparser
 import codecs
 import flask
@@ -55,16 +55,16 @@ def get_version_info(module):
     """
     module = module.lower()
 
-    def cherrypy_ver():
-        import cherrypy
-        return cherrypy.__version__
+    def cheroot_ver():
+        import cheroot
+        return cheroot.__version__
 
     def sherlock_ver():
         from . import sherlock
         return sherlock.__version__
 
     return {
-        'cherrypy': cherrypy_ver,
+        'cheroot': cheroot_ver,
         'whoosh': whoosh.versionstring,
         'pygments': lambda: pygments.__version__,
         'flask': lambda: flask.__version__,

core/sherlock/__init__.py

@@ -6,7 +6,7 @@
 refs:
 http://docs.python.org/3/library/logging.html
 """
-__version__ = '0.7.3'
+__version__ = '0.8.0'
 
 import logging
 logger = logging.getLogger('core.sherlock')

core/sherlock/indexer.py

@@ -11,6 +11,7 @@
 import os
 import settings
 import shutil
+import contextlib
 from core import FULL_INDEXES_PATH, FORCE_INDEX_REBUILD
 from core.sherlock import logger as log
 from core.sherlock import searcher
@@ -39,7 +40,7 @@ def index_paths(paths, name=settings.DEFAULT_INDEX_NAME):
     # index files for the search
     with _get_indexer_with_cleanup(name) as idxr:
         for path in paths:
-            idxr.index_text(unicode(path))
+            idxr.index_text(path)
 
 
 def index_path(path, name=settings.DEFAULT_INDEX_NAME):
@@ -51,7 +52,7 @@ def index_path(path, name=settings.DEFAULT_INDEX_NAME):
     """
     # index a file for the search
     with _get_indexer_with_cleanup(name) as idxr:
-        idxr.index_text(unicode(path))
+        idxr.index_text(path)
 
 
 @contextlib.contextmanager

example.local_settings.yml

@@ -18,5 +18,5 @@ include_file_suffix: [
 
 # The name of the server type to use as the web server.
 # type: string
-# default: '' or 'cherrypy'
-server_type: 'cherrypy'
+# default: '' or 'cheroot'
+server_type: 'cheroot'

main.py

@@ -28,7 +28,7 @@ def show_version():
     print('   Flask: v' + get_version_info('flask'))
     print('Pygments: v' + get_version_info('pygments'))
     print('  Whoosh: v' + get_version_info('whoosh'))
-    print('CherryPy: v' + get_version_info('cherrypy'))
+    print(' Cheroot: v' + get_version_info('cheroot'))
 
 
 def show_stats():
@@ -47,8 +47,8 @@ def run_server():
     if settings.LOG_PATH:
         print('Log path: %s' % settings.LOG_PATH)
     print('Backend: %s' % settings.DEFAULT_SEARCHER)
-    print('Server: %s' % settings.SERVER_TYPE)
-    print('Listening on: %s:%d' % (settings.SERVER_ADDRESS, settings.SERVER_PORT))
+    print('Server: %s' % (settings.SERVER_TYPE or 'development'))
+    print('Listening on: {}:{}'.format(settings.SERVER_ADDRESS, settings.SERVER_PORT))
     # launch web server
     server.run()
 
@@ -65,7 +65,7 @@ def reindex():
     if FORCE_INDEX_REBUILD:
         wait_time = 5  # seconds to wait/pause until rebuilding index
         print('Reindexing everything!')
-        print('Waiting %ss for interrupt...' % wait_time)
+        print('Waiting {}s for interrupt...'.format(wait_time))
         import time
         time.sleep(wait_time)
     print('Indexing started.')

settings.py

@@ -101,7 +101,7 @@
 DEFAULT_INDEX_NAME = config.get('default_index_name', 'main')
 
 # The name of the server type to use as the web server.
-# CherryPy support is built-in, if production: 'cherrypy'.
+# Cheroot support is built-in, if production: 'cheroot'.
 # type: string
 # default: None
 SERVER_TYPE = config.get('server_type')
@@ -168,12 +168,3 @@
 # type: string
 # default: black
 SITE_BANNER_COLOR = config.get('site_banner_color', 'black')
-
-
-# Use the local_settings.yml instead, noted at the top of file
-try:
-    from local_settings import *
-    print('!!!Deprecated local_settings.py|pyc file found: Use local_settings.yml instead.')
-except ImportError:
-    # ignore import error, because it's deprecated
-    pass

setup/requirements.txt

@@ -1,9 +1,9 @@
-flask>=0.10,<=0.11.1
-whoosh<2.7.0
-jinja2==2.8
-pygments<=2.1.3
-cherrypy>5.0.0,<=8.1.2
-flask-peewee<=0.6.7
-peewee<=2.8.5
+flask==1.1.1
+whoosh==2.7.4
+jinja2==2.11.1
+pygments==2.5.2
+cheroot==8.3.0
+flask-peewee==3.0.3
+peewee==3.13.1
 path.py
 pyyaml

setup/virtualenv-setup.sh

@@ -1,30 +1,40 @@
 # global
 # Created by: Christopher Bess
-# It is recomm. not to run this script using sudo, it is used as needed.
+# It is NOT recommended to run this script using sudo, it is used as needed.
 #
 # user$: sh virtualenv-setup.sh
 
-echo "Setting up virtualenv and pip - installing with sudo"
-sudo easy_install pip
-sudo pip install virtualenv
+if ! [ -x "$(command -v python3)" ]; then
+    echo "Python 3+ required."
+    exit 1
+fi
 
-PROJROOT=$(cd $(dirname $0) && cd .. && pwd)
+if ! [ -x "$(command -v pip)" ]; then
+    echo "Setting up pip - installing with sudo"
+
+    sudo easy_install pip
+fi
+
+PROJ_ROOT=$(cd $(dirname $0) && cd .. && pwd)
 
 # adjust permission (allow it to be executed)
-chmod +x ${PROJROOT}/main.py
+chmod +x ${PROJ_ROOT}/main.py
 
 # if on a Mac exec below line (maybe)
 # ARCHFLAGS="-arch i386 -arch x86_64"
 
+VENV_ROOT=${PROJ_ROOT}/sherlock_env
+
 # setup sherlock environment
-mkdir -p ${PROJROOT}/data/indexes
-virtualenv ${PROJROOT}/sherlock_env --distribute --no-site-packages
+mkdir -p ${PROJ_ROOT}/data/indexes
+python3 -m venv ${VENV_ROOT}
 
 echo "Installing sherlock dependencies"
-${PROJROOT}/sherlock_env/bin/pip install -r ${PROJROOT}/setup/requirements.txt
+${VENV_ROOT}/bin/pip install --upgrade pip
+${VENV_ROOT}/bin/pip install -r ${PROJ_ROOT}/setup/requirements.txt
 
 # confirm installation by showing version information
 # echo "Sherlock version information"
-${PROJROOT}/sherlock_env/bin/python ${PROJROOT}/main.py -v
+${VENV_ROOT}/bin/python ${PROJ_ROOT}/main.py -v
 
 echo "Sherlock install finished"

tests/test_searcher.py

@@ -76,7 +76,7 @@ def test_suggestions(self):
         search_text = 'var'
         result = idx.suggestions(search_text)
         self.assertTrue(result, 'no suggestions returned')
-        self.assertIn('val', result, 'suggestion not matching')
+        self.assertIn('char', result, 'suggestion not matching')
         self.assertNotIn(search_text, result, 'original query should not be included')
 
 

tests/testcase.py

@@ -26,7 +26,7 @@ class BaseTestCase(unittest.TestCase):
     """ The base class for all Sherlock test cases """
 
     def setUp(self):
-        self.test_dir = '%s/tests' % settings.ROOT_DIR
+        self.test_dir = os.path.join(settings.ROOT_DIR, 'tests')
         settings.DEFAULT_INDEX_NAME = 'test'
 
     def get_test_string(self):
@@ -37,4 +37,4 @@ def get_test_string(self):
 
         @return: a string of the current time
         """
-        return "current time tuple: %s" % time.localtime()
+        return 'current time tuple: %s' % time.localtime()

webapp/server_cherrypy.py → webapp/server_cheroot.py

@@ -1,20 +1,19 @@
-# Cherrypy server add-on
-# Install: pip install cherrypy
+# Cheroot server add-on
+# Install: pip install cheroot
 # refs:
 # http://stackoverflow.com/questions/4884541/cherrypy-vs-flask-werkzeug
 # https://github.com/radekstepan/Flask-Skeleton-App
 # http://stackoverflow.com/questions/5982638/using-cherrypy-cherryd-to-launch-multiple-flask-instances
 # http://flask.pocoo.org/snippets/24/
-# http://docs.cherrypy.org/dev/refman/wsgiserver/init.html
-from core import cherrypy_wsgiserver
+# https://docs.cherrypy.org/projects/cheroot/en/latest/pkg/cheroot.wsgi.html
+from core import WSGIPathInfoDispatcher, WSGIServer
 from .server import app
 from core import settings as core_settings
 
-# setup cherrypy server
-dispatcher = cherrypy_wsgiserver.WSGIPathInfoDispatcher({'/': app})
-server = cherrypy_wsgiserver.CherryPyWSGIServer(
+# setup cheroot server
+server = WSGIServer(
     (core_settings.SERVER_ADDRESS, core_settings.SERVER_PORT),
-    dispatcher,
+    WSGIPathInfoDispatcher({'/': app}),
     server_name='text.sherlock',
     numthreads=10  # default: 10
 )