# -*- coding: utf-8 -*- """ All we need to create a Solr connection is a url. >>> conn = Solr('http://127.0.0.1:8983/solr/') First, completely clear the index. >>> conn.delete(q='*:*') For now, we can only index python dictionaries. Each key in the dictionary will correspond to a field in Solr. >>> docs = [ ... {'id': 'testdoc.1', 'order_i': 1, 'name': 'document 1', 'text': u'Paul Verlaine'}, ... {'id': 'testdoc.2', 'order_i': 2, 'name': 'document 2', 'text': u'Владимир Маякoвский'}, ... {'id': 'testdoc.3', 'order_i': 3, 'name': 'document 3', 'text': u'test'}, ... {'id': 'testdoc.4', 'order_i': 4, 'name': 'document 4', 'text': u'test'} ... ] We can add documents to the index by passing a list of docs to the connection's add method. >>> conn.add(docs) >>> results = conn.search('Verlaine') >>> len(results) 1 >>> results = conn.search(u'Владимир') >>> len(results) 1 Simple tests for searching. We can optionally sort the results using Solr's sort syntax, that is, the field name and either asc or desc. >>> results = conn.search('test', sort='order_i asc') >>> for result in results: ... print result['name'] document 3 document 4 >>> results = conn.search('test', sort='order_i desc') >>> for result in results: ... print result['name'] document 4 document 3 To update documents, we just use the add method. >>> docs = [ ... {'id': 'testdoc.4', 'order_i': 4, 'name': 'document 4', 'text': u'blah'} ... ] >>> conn.add(docs) >>> len(conn.search('blah')) 1 >>> len(conn.search('test')) 1 We can delete documents from the index by id, or by supplying a query. >>> conn.delete(id='testdoc.1') >>> conn.delete(q='name:"document 2"') >>> results = conn.search('Verlaine') >>> len(results) 0 Docs can also have multiple values for any particular key. This lets us use Solr's multiValue fields. >>> docs = [ ... {'id': 'testdoc.5', 'cat': ['poetry', 'science'], 'name': 'document 5', 'text': u''}, ... {'id': 'testdoc.6', 'cat': ['science-fiction',], 'name': 'document 6', 'text': u''}, ... ] >>> conn.add(docs) >>> results = conn.search('cat:"poetry"') >>> for result in results: ... print result['name'] document 5 >>> results = conn.search('cat:"science-fiction"') >>> for result in results: ... print result['name'] document 6 >>> results = conn.search('cat:"science"') >>> for result in results: ... print result['name'] document 5 Docs can also boost any particular key. This lets us use Solr's boost on a field. >>> docs = [ ... {'id': 'testdoc.7', 'order_i': '7', 'name': 'document 7', 'text': u'eight', 'author': 'seven'}, ... {'id': 'testdoc.8', 'order_i': '8', 'name': 'document 8', 'text': u'seven', 'author': 'eight'}, ... ] >>> conn.add(docs, boost={'author': '2.0',}) >>> results = conn.search('seven author:seven') >>> for result in results: ... print result['name'] document 7 document 8 >>> results = conn.search('eight author:eight') >>> for result in results: ... print result['name'] document 8 document 7 """ # TODO: unicode support is pretty sloppy. define it better. from datetime import datetime import logging import re import time import urllib import urllib2 from urlparse import urlsplit, urlunsplit try: # for python 2.5 from xml.etree import cElementTree as ET except ImportError: try: # use etree from lxml if it is installed from lxml import etree as ET except ImportError: try: # use cElementTree if available import cElementTree as ET except ImportError: try: from elementtree import ElementTree as ET except ImportError: raise ImportError("No suitable ElementTree implementation was found.") try: # For Python < 2.6 or people using a newer version of simplejson import simplejson as json except ImportError: # For Python >= 2.6 import json try: # Desirable from a timeout perspective. from httplib2 import Http TIMEOUTS_AVAILABLE = True except ImportError: from httplib import HTTPConnection TIMEOUTS_AVAILABLE = False try: set except NameError: from sets import Set as set __author__ = 'Joseph Kocherhans, Jacob Kaplan-Moss, Daniel Lindsley' __all__ = ['Solr'] __version__ = (2, 0, 14) def get_version(): return "%s.%s.%s" % __version__[:3] DATETIME_REGEX = re.compile('^(?P\d{4})-(?P\d{2})-(?P\d{2})T(?P\d{2}):(?P\d{2}):(?P\d{2})(\.\d+)?Z$') class NullHandler(logging.Handler): def emit(self, record): pass # Add the ``NullHandler`` to avoid logging by default while still allowing # others to attach their own handlers. LOG = logging.getLogger('pysolr') h = NullHandler() LOG.addHandler(h) # For debugging... if False: LOG.setLevel(logging.DEBUG) stream = logging.StreamHandler() LOG.addHandler(stream) def safe_urlencode(params, doseq=0): """ UTF-8-safe version of safe_urlencode The stdlib safe_urlencode prior to Python 3.x chokes on UTF-8 values which can't fail down to ascii. """ if hasattr(params, "items"): params = params.items() new_params = list() for k, v in params: k = k.encode("utf-8") if isinstance(v, basestring): new_params.append((k, v.encode("utf-8"))) elif isinstance(v, (list, tuple)): new_params.append((k, [i.encode("utf-8") for i in v])) else: new_params.append((k, unicode(v))) return urllib.urlencode(new_params, doseq) class SolrError(Exception): pass class Results(object): def __init__(self, docs, hits, highlighting=None, facets=None, spellcheck=None, stats=None): self.docs = docs self.hits = hits self.highlighting = highlighting or {} self.facets = facets or {} self.spellcheck = spellcheck or {} self.stats = stats or {} def __len__(self): return len(self.docs) def __iter__(self): return iter(self.docs) class Solr(object): def __init__(self, url, decoder=None, timeout=60): self.decoder = decoder or json.JSONDecoder() self.url = url self.scheme, netloc, path, query, fragment = urlsplit(url) self.base_url = urlunsplit((self.scheme, netloc, '', '', '')) netloc = netloc.split(':') self.host = netloc[0] if len(netloc) == 1: self.host, self.port = netloc[0], None else: self.host, self.port = netloc[0], int(netloc[1]) self.path = path.rstrip('/') self.timeout = timeout self.log = self._get_log() def _get_log(self): return LOG def _send_request(self, method, path, body=None, headers=None): if TIMEOUTS_AVAILABLE: http = Http(timeout=self.timeout) url = self.base_url + path try: start_time = time.time() self.log.debug("Starting request to '%s' (%s) with body '%s'..." % (url, method, str(body)[:10])) headers, response = http.request(url, method=method, body=body, headers=headers) end_time = time.time() self.log.info("Finished '%s' (%s) with body '%s' in %0.3f seconds." % (url, method, str(body)[:10], end_time - start_time)) except AttributeError: # For httplib2. error_message = "Failed to connect to server at '%s'. Are you sure '%s' is correct? Checking it in a browser might help..." % (url, self.base_url) self.log.error(error_message) raise SolrError(error_message) if int(headers['status']) != 200: error_message = self._extract_error(headers, response) self.log.error(error_message) raise SolrError(error_message) return response else: if headers is None: headers = {} conn = HTTPConnection(self.host, self.port) start_time = time.time() self.log.debug("Starting request to '%s:%s/%s' (%s) with body '%s'..." % (self.host, self.port, path, method, str(body)[:10])) conn.request(method, path, body, headers) response = conn.getresponse() end_time = time.time() self.log.info("Finished '%s:%s/%s' (%s) with body '%s' in %0.3f seconds." % (self.host, self.port, path, method, str(body)[:10], end_time - start_time)) if response.status != 200: error_message = self._extract_error(dict(response.getheaders()), response.read()) self.log.error(error_message) raise SolrError(error_message) return response.read() def _select(self, params): # specify json encoding of results params['wt'] = 'json' params_encoded = safe_urlencode(params, True) if len(params_encoded) < 1024: # Typical case. path = '%s/select/?%s' % (self.path, params_encoded) return self._send_request('GET', path) else: # Handles very long queries by submitting as a POST. path = '%s/select/' % (self.path,) headers = { 'Content-type': 'application/x-www-form-urlencoded; charset=utf-8', } return self._send_request('POST', path, body=params_encoded, headers=headers) def _mlt(self, params): params['wt'] = 'json' # specify json encoding of results path = '%s/mlt/?%s' % (self.path, safe_urlencode(params, True)) return self._send_request('GET', path) def _suggest_terms(self, params): params['wt'] = 'json' # specify json encoding of results path = '%s/terms/?%s' % (self.path, safe_urlencode(params, True)) return self._send_request('GET', path) def _update(self, message, clean_ctrl_chars=True, commit=True, waitFlush=None, waitSearcher=None): """ Posts the given xml message to http://:/solr/update and returns the result. Passing `sanitize` as False will prevent the message from being cleaned of control characters (default True). This is done by default because these characters would cause Solr to fail to parse the XML. Only pass False if you're positive your data is clean. """ path = '%s/update/' % self.path # Per http://wiki.apache.org/solr/UpdateXmlMessages, we can append a # ``commit=true`` to the URL and have the commit happen without a # second request. query_vars = [] if commit is not None: query_vars.append('commit=%s' % str(bool(commit)).lower()) if waitFlush is not None: query_vars.append('waitFlush=%s' % str(bool(waitFlush)).lower()) if waitSearcher is not None: query_vars.append('waitSearcher=%s' % str(bool(waitSearcher)).lower()) if query_vars: path = '%s?%s' % (path, '&'.join(query_vars)) # Clean the message of ctrl characters. if clean_ctrl_chars: message = sanitize(message) return self._send_request('POST', path, message, {'Content-type': 'text/xml; charset=utf-8'}) def _extract_error(self, headers, response): """ Extract the actual error message from a solr response. """ reason = headers.get('reason', None) full_html = None if reason is None: reason, full_html = self._scrape_response(headers, response) msg = "[Reason: %s]" % reason if reason is None: msg += "\n%s" % full_html return msg def _scrape_response(self, headers, response): """ Scrape the html response. """ # identify the responding server server_type = None server_string = headers.get('server', '') if 'jetty' in server_string.lower(): server_type = 'jetty' if 'coyote' in server_string.lower(): # TODO: During the pysolr 3 effort, make this no longer a # conditional and consider using ``lxml.html`` instead. from BeautifulSoup import BeautifulSoup server_type = 'tomcat' reason = None full_html = '' dom_tree = None if server_type == 'tomcat': # Tomcat doesn't produce a valid XML response soup = BeautifulSoup(response) body_node = soup.find('body') p_nodes = body_node.findAll('p') for p_node in p_nodes: children = p_node.findChildren() if len(children) >= 2 and 'message' in children[0].renderContents().lower(): reason = children[1].renderContents() if reason is None: full_html = soup.prettify() else: # Let's assume others do produce a valid XML response try: dom_tree = ET.fromstring(response) reason_node = None # html page might be different for every server if server_type == 'jetty': reason_node = dom_tree.find('body/pre') if reason_node is not None: reason = reason_node.text if reason is None: full_html = ET.tostring(dom_tree) except SyntaxError, e: full_html = "%s" % response full_html = full_html.replace('\n', '') full_html = full_html.replace('\r', '') full_html = full_html.replace('
', '') full_html = full_html.replace('
', '') full_html = full_html.strip() return reason, full_html # Conversion ############################################################# def _from_python(self, value): """ Converts python values to a form suitable for insertion into the xml we send to solr. """ if hasattr(value, 'strftime'): if hasattr(value, 'hour'): value = "%sZ" % value.isoformat() else: value = "%sT00:00:00Z" % value.isoformat() elif isinstance(value, bool): if value: value = 'true' else: value = 'false' elif isinstance(value, str): value = unicode(value, errors='replace') else: value = unicode(value) return value def _to_python(self, value): """ Converts values from Solr to native Python values. """ if isinstance(value, (int, float, long, complex)): return value if isinstance(value, (list, tuple)): value = value[0] if value == 'true': return True elif value == 'false': return False if isinstance(value, basestring): possible_datetime = DATETIME_REGEX.search(value) if possible_datetime: date_values = possible_datetime.groupdict() for dk, dv in date_values.items(): date_values[dk] = int(dv) return datetime(date_values['year'], date_values['month'], date_values['day'], date_values['hour'], date_values['minute'], date_values['second']) try: # This is slightly gross but it's hard to tell otherwise what the # string's original type might have been. Be careful who you trust. converted_value = eval(value) # Try to handle most built-in types. if isinstance(converted_value, (list, tuple, set, dict, int, float, long, complex)): return converted_value except: # If it fails (SyntaxError or its ilk) or we don't trust it, # continue on. pass return value def _is_null_value(self, value): """ Check if a given value is ``null``. Criteria for this is based on values that shouldn't be included in the Solr ``add`` request at all. """ # TODO: This should probably be removed when solved in core Solr level? return (value is None) or (isinstance(value, basestring) and len(value) == 0) # API Methods ############################################################ def search(self, q, **kwargs): """Performs a search and returns the results.""" params = {'q': q} params.update(kwargs) response = self._select(params) # TODO: make result retrieval lazy and allow custom result objects result = self.decoder.decode(response) result_kwargs = {} if result.get('highlighting'): result_kwargs['highlighting'] = result['highlighting'] if result.get('facet_counts'): result_kwargs['facets'] = result['facet_counts'] if result.get('spellcheck'): result_kwargs['spellcheck'] = result['spellcheck'] if result.get('stats'): result_kwargs['stats'] = result['stats'] self.log.debug("Found '%s' search results." % result['response']['numFound']) return Results(result['response']['docs'], result['response']['numFound'], **result_kwargs) def more_like_this(self, q, mltfl, **kwargs): """ Finds and returns results similar to the provided query. Requires Solr 1.3+. """ params = { 'q': q, 'mlt.fl': mltfl, } params.update(kwargs) response = self._mlt(params) result = self.decoder.decode(response) if result['response'] is None: result['response'] = { 'docs': [], 'numFound': 0, } self.log.debug("Found '%s' MLT results." % result['response']['numFound']) return Results(result['response']['docs'], result['response']['numFound']) def suggest_terms(self, fields, prefix, **kwargs): """ Accepts a list of field names and a prefix Returns a dictionary keyed on field name containing a list of ``(term, count)`` pairs Requires Solr 1.4+. """ params = { 'terms.fl': fields, 'terms.prefix': prefix, } params.update(kwargs) response = self._suggest_terms(params) result = self.decoder.decode(response) terms = result.get("terms", {}) res = {} while terms: # The raw values are a flat list: ["dance",23,"dancers",10,"dancing",8,"dancer",6]] field = terms.pop(0) values = terms.pop(0) tmp = list() while values: tmp.append((values.pop(0), values.pop(0))) res[field] = tmp self.log.debug("Found '%d' Term suggestions results.", sum(len(j) for i, j in res.items())) return res def add(self, docs, commit=True, boost=None, commitWithin=None, waitFlush=None, waitSearcher=None): """Adds or updates documents. For now, docs is a list of dictionaries where each key is the field name and each value is the value to index. """ start_time = time.time() self.log.debug("Starting to build add request...") message = ET.Element('add') if commitWithin: message.set('commitWithin', commitWithin) for doc in docs: d = ET.Element('doc') for key, value in doc.items(): if key == 'boost': d.set('boost', str(value)) continue # handle lists, tuples, and other iterables if hasattr(value, '__iter__'): for v in value: if self._is_null_value(value): continue if boost and v in boost: if not isinstance(boost, basestring): boost[v] = str(boost[v]) f = ET.Element('field', name=key, boost=boost[v]) else: f = ET.Element('field', name=key) f.text = self._from_python(v) d.append(f) # handle strings and unicode else: if self._is_null_value(value): continue if boost and key in boost: if not isinstance(boost, basestring): boost[key] = str(boost[key]) f = ET.Element('field', name=key, boost=boost[key]) else: f = ET.Element('field', name=key) f.text = self._from_python(value) d.append(f) message.append(d) m = ET.tostring(message, encoding='utf-8') end_time = time.time() self.log.debug("Built add request of %s docs in %0.2f seconds." % (len(docs), end_time - start_time)) response = self._update(m, commit=commit, waitFlush=waitFlush, waitSearcher=waitSearcher) def delete(self, id=None, q=None, commit=True, waitFlush=None, waitSearcher=None): """Deletes documents.""" if id is None and q is None: raise ValueError('You must specify "id" or "q".') elif id is not None and q is not None: raise ValueError('You many only specify "id" OR "q", not both.') elif id is not None: m = '%s' % id elif q is not None: m = '%s' % q response = self._update(m, commit=commit, waitFlush=waitFlush, waitSearcher=waitSearcher) def commit(self, waitFlush=None, waitSearcher=None, expungeDeletes=None): if expungeDeletes is not None: msg = '' % str(bool(expungeDeletes)).lower() else: msg = '' response = self._update(msg, waitFlush=waitFlush, waitSearcher=waitSearcher) def optimize(self, waitFlush=None, waitSearcher=None, maxSegments=None): if maxSegments: msg = '' % maxSegments else: msg = '' response = self._update('', waitFlush=waitFlush, waitSearcher=waitSearcher) class SolrCoreAdmin(object): """ Handles core admin operations: see http://wiki.apache.org/solr/CoreAdmin Operations offered by Solr are: 1. STATUS 2. CREATE 3. RELOAD 4. RENAME 5. ALIAS 6. SWAP 7. UNLOAD 8. LOAD (not currently implemented) """ def __init__(self, url, *args, **kwargs): super(SolrCoreAdmin, self).__init__(*args, **kwargs) self.url = url def _get_url(self, url, params={}, headers={}): request = urllib2.Request(url, data=safe_urlencode(params), headers=headers) # Let ``socket.error``, ``urllib2.HTTPError`` and ``urllib2.URLError`` # propagate up the stack. response = urllib2.urlopen(request) return response.read() def status(self, core=None): """http://wiki.apache.org/solr/CoreAdmin#head-9be76f5a459882c5c093a7a1456e98bea7723953""" params = { 'action': 'STATUS', } if core is not None: params.update(core=core) return self._get_url(self.url, params=params) def create(self, name, instance_dir=None, config='solrcofig.xml', schema='schema.xml'): """http://wiki.apache.org/solr/CoreAdmin#head-7ca1b98a9df8b8ca0dcfbfc49940ed5ac98c4a08""" params = { 'action': 'STATUS', 'name': name, 'config': config, 'schema': schema, } if instance_dir is None: params.update(instanceDir=name) else: params.update(instanceDir=instance_dir) return self._get_url(self.url, params=params) def reload(self, core): """http://wiki.apache.org/solr/CoreAdmin#head-3f125034c6a64611779442539812067b8b430930""" params = { 'action': 'RELOAD', 'core': core, } return self._get_url(self.url, params=params) def rename(self, core, other): """http://wiki.apache.org/solr/CoreAdmin#head-9473bee1abed39e8583ba45ef993bebb468e3afe""" params = { 'action': 'RENAME', 'core': core, 'other': other, } return self._get_url(self.url, params=params) def alias(self, core, other): """ http://wiki.apache.org/solr/CoreAdmin#head-8bf9004eaa4d86af23d2758aafb0d31e2e8fe0d2 Experimental feature in Solr 1.3 """ params = { 'action': 'ALIAS', 'core': core, 'other': other, } return self._get_url(self.url, params=params) def swap(self, core, other): """http://wiki.apache.org/solr/CoreAdmin#head-928b872300f1b66748c85cebb12a59bb574e501b""" params = { 'action': 'SWAP', 'core': core, 'other': other, } return self._get_url(self.url, params=params) def unload(self, core): """http://wiki.apache.org/solr/CoreAdmin#head-f5055a885932e2c25096a8856de840b06764d143""" params = { 'action': 'UNLOAD', 'core': core, } return self._get_url(self.url, params=params) def load(self, core): raise NotImplementedError('Solr 1.4 and below do not support this operation.') # Using two-tuples to preserve order. REPLACEMENTS = ( # Nuke nasty control characters. ('\x00', ''), # Start of heading ('\x01', ''), # Start of heading ('\x02', ''), # Start of text ('\x03', ''), # End of text ('\x04', ''), # End of transmission ('\x05', ''), # Enquiry ('\x06', ''), # Acknowledge ('\x07', ''), # Ring terminal bell ('\x08', ''), # Backspace ('\x0b', ''), # Vertical tab ('\x0c', ''), # Form feed ('\x0e', ''), # Shift out ('\x0f', ''), # Shift in ('\x10', ''), # Data link escape ('\x11', ''), # Device control 1 ('\x12', ''), # Device control 2 ('\x13', ''), # Device control 3 ('\x14', ''), # Device control 4 ('\x15', ''), # Negative acknowledge ('\x16', ''), # Synchronous idle ('\x17', ''), # End of transmission block ('\x18', ''), # Cancel ('\x19', ''), # End of medium ('\x1a', ''), # Substitute character ('\x1b', ''), # Escape ('\x1c', ''), # File separator ('\x1d', ''), # Group separator ('\x1e', ''), # Record separator ('\x1f', ''), # Unit separator ) def sanitize(data): fixed_string = data for bad, good in REPLACEMENTS: fixed_string = fixed_string.replace(bad, good) return fixed_string if __name__ == "__main__": import doctest doctest.testmod()