xml output

internal links.

This doesn't make much difference, but I think it is nicer to assume bad
URLs are external, rather than internal.

.gitignore

@@ -0,0 +1,31 @@
+^JAVA_CMD$
+^PYTHON_CMD$
+
+.DS_Store
+
+# ignore any data
+data
+
+# ignore working bytecode
+\.class$
+\.pyc$
+
+^datadump/*
+
+# ignore test and tutorial directories
+test/*$
+tests/*$
+testsuite/*$
+tutorial/*$
+
+# Solr/Jetty
+^java/solr/server/solr-webapp/*
+^java/solr/server/logs/*
+
+# ignore the downloaded logos
+^python/media/images/logos48
+^python/media/images/icons
+^python/media/images/badges
+
+# PyCharm project files
+^.idea/

.hgignore

@@ -22,3 +22,6 @@ tutorial/.*$
 
 # ignore the downloaded logos
 ^python/media/images/logos/.*
+
+# PyCharm project files
+^.idea/

.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/stackdump.iml" filepath="$PROJECT_DIR$/.idea/stackdump.iml" />
+    </modules>
+  </component>
+</project>

.idea/stackdump.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

README.textile

@@ -1,3 +1,8 @@
+Tutorial (Thai language) : http://www.youtube.com/watch?v=jHEBlHxEKeM
+"http://blog.sornram9254.com/stackdump-an-offline-browser-for-stackexchange-stackverflow/":http://blog.sornram9254.com/stackdump-an-offline-browser-for-stackexchange-stackverflow/
+
+- - - - - - - - - - - - - - - - - - - ORIGINAL README - - - - - - - - - - - - - - - - - - -
+
 h1. Stackdump - an offline browser for StackExchange sites.
 
 Stackdump was conceived for those who work in environments that do not have easy access to the StackExchange family of websites. It allows you to host a read-only instance of the StackExchange sites locally, accessible via a web browser.
@@ -12,13 +17,15 @@ h2. Screenshots
 
 h2. System Requirements
 
-Stackdump was written in Python and requires Python 2.5 or later (but not Python 3). It leverages Apache Solr, which requires the Java runtime (JRE), version 6 or later.
+Stackdump was written in Python and requires Python 2.5 or later (but not Python 3). There have been some reported issues with older versions of Python, so the ideal version to run is v2.7.6 (the latest 2.x version, as of writing). Stackdump also leverages Apache Solr, which requires the Java runtime (JRE), version 6 or later.
 
 Besides that, there are no OS-dependent dependencies and should work on any platform that Python and Java run on (although it only comes bundled with Linux scripts at the moment). It was, however, developed and tested on CentOS 5 running Python 2.7 and JRE 6 update 27.
 
 You will also need "7-zip":http://www.7-zip.org/ to extract the data dump files, but Stackdump does not use it directly so you can perform the extraction on another machine first.
 
-It is recommended that Stackdump be run on a system with at least 3GB of RAM, particularly if you intend to import StackOverflow into Stackdump. Apache Solr requires a fair bit of memory during the import process. It should also have a fair bit of space available; having at least roughly the space used by the raw, extracted, data dump XML files is a good rule of thumb (note that once imported, the raw data dump XML files are not needed by Stackdump any more).
+The amount of memory required for Stackdump depends on which dataset you want to import. For most datasets, at least 3GB of RAM is preferable. If you want to import StackOverflow, you must use a 64-bit operating system and a 64-bit version of Python, and also have at least 6GB of RAM available (or swap). If you do not have enough RAM available, the import process will likely fail with a _MemoryError_ message at some point.
+
+Make sure you have enough disk space too - having at least roughly the space used by the raw, extracted, data dump XML files available is a good rule of thumb (note that once imported, the raw data dump XML files are not needed by Stackdump any more).
 
 Finally, Stackdump has been tested and works in the latest browsers (IE9, FF10+, Chrome, Safari). It degrades fairly gracefully in older browsers, although some will have rendering issues, e.g. IE8.
 
@@ -28,24 +35,59 @@ Version 1.1 fixes a few bugs, the major one being the inability to import the 20
 
 Because changes have been made to the search schema and the search indexer has been upgraded (to Solr 4.5), all data will need to be re-indexed. Therefore there is no upgrade path; follow the instructions below to set up Stackdump again. It is recommended to install this new version in a new directory, instead of overwriting the existing one.
 
+h2. Changes and upgrading from v1.1 to v1.2
+
+The major change in the v1.2 release are improvements to the speed of importing data. There are some other smaller changes, including new PowerShell scripts to start and manage Stackdump on Windows as well as a few bug fixes when running on Windows. The search indexing side of things has not changed, therefore data imported using v1.1 will continue to work in v1.2. _Data from older versions however, needs to be re-indexed. See the above section on upgrading to v1.1 for more details._
+
+h2. Changes and upgrading from v1.2 to v1.3
+
+v1.3 is primarily bugfix release, for a fairly serious bug. It turns out Stackdump has been subtly overwriting questions as more sites are imported because it assumed post IDs were unique across all sites, when they in fact were not.  This meant as more sites were imported, the previous sites started to lose questions. The fix required a change to search index, therefore *the data directory will need to be deleted and all data will need to be re-imported after installing this version*. Thanks to @yammesicka for reporting the issue.
+
+Other changes include a new setting to allow disabling the link and image URL rewriting, and a change to the @import_site@ command so it doesn't bail immediately if there is a Solr connection issue - it will prompt and allow resumption after the connection issue has been resolved.
+
+h3. Importing the StackOverflow data dump, September 2013
+
+The StackOverflow data dump has grown significantly since I started this project back in 2011. With the improvements in v1.2, on a VM with two cores and 4GB of RAM running CentOS 5.7 on a single, standard hard drive containing spinning pieces of metal,
+
+* it took *84719.565491 seconds* to import it, or *23 hours, 31 minutes and 59.565491 seconds*
+* once completed, it used up *20GB* of disk space
+* during the import, roughly *30GB* of disk space was needed
+* the import process used, at max, around *2GB* of RAM.
+
+In total, the StackOverflow data dump has *15,933,529 posts* (questions and answers), *2,332,403 users* and a very large number of comments.
+
+I attempted this on a similarly spec'ed Windows 7 64-bit VM as well - 23 hours later and it is still trying to process the comments. The SQLite, Python or just disk performance is very poor for some reason. Therefore, if you intend on importing StackOverflow, I would advise you to run Stackdump on Linux instead. The smaller sites all complete without a reasonable time though, and there are no perceptible issues with performance as far as I'm aware on Windows.
+
+h3. Reports on importing the StackOverflow data dump, September 2014
+
+Due to the growth of the dataset, the import process now requires at least 6GB of RAM. This also means you must use a 64-bit operating system and a 64-bit version of Python.
+
 h2. Setting up
 
 Stackdump was designed for offline environments or environments with poor internet access, therefore it is bundled with all the dependencies it requires (with the exception of Python, Java and 7-zip). 
 
 As long as you have:
-* "Python":http://python.org/download/,
-* "Java":http://java.com/en/download/manual.jsp,
+* "Python":http://python.org/download/, version 2.5 or later but not version 3 (tested with v2.7.6),
+* "Java":http://java.com/en/download/manual.jsp, version 6 (1.6) or later,
 * "Stackdump":https://bitbucket.org/samuel.lai/stackdump/downloads,
-* the "StackExchange Data Dump":http://www.clearbits.net/creators/146-stack-exchange-data-dump (Note: this is only available as a torrent), and
+* the "StackExchange Data Dump":https://archive.org/details/stackexchange (download the sites you wish to import - note that StackOverflow is split into 7 archive files; only Comments, Posts and Users are required but after extraction the files need to be renamed to Comments.xml, Posts.xml and Users.xml respectively), and
 * "7-zip":http://www.7-zip.org/ (needed to extract the data dump files)
 
 ...you should be able to get an instance up and running.
 
+If you are using a 64-bit operating system, get the 64-bit version of Python.
+
 To provide a better experience, Stackdump can use the RSS feed content to pre-fill some of the required details during the import process, as well as to display the site logos in the app. Stackdump comes bundled with a script that downloads and places these bits in the right places. If you're in a completely offline environment however, it may be worth running this script on a connected box first.
 
+h3. Windows users
+
+If you're using Windows, you will need to substitute the appropriate PowerShell equivalent command for the Stackdump scripts used below. These equivalent PowerShell scripts are in the Stackdump root directory, alongside their Unix counterparts. The names are roughly the same, with the exception of @manage.sh@, which in PowerShell has been broken up into two scripts, @List-StackdumpCommands.ps1@ and @Run-StackdumpCommand.ps1@.
+
+Remember to set your PowerShell execution policy to at least @RemoteSigned@ first as these scripts are not signed. Use the @Get-ExecutionPolicy@ cmdlet to see the current policy, and @Set-ExecutionPolicy@ to set it. You will need to have administrative privileges to set it. 
+
 h3. Extract Stackdump
 
-Stackdump was to be self-contained, so to get it up and running, simply extract the Stackdump download to an appropriate location.
+Stackdump was designed to be self-contained, so to get it up and running, simply extract the Stackdump download archive to an appropriate location.
 
 h3. Verify dependencies
 
@@ -58,7 +100,7 @@ bq. If you're using Java 7 on Linux and you see an error similar to the followin
 this is because you have SELinux enabled. You will need to tell SELinux to allow Java to run by using the following command as root (amending the path as necessary) -
 @chcon -t textrel_shlib_t /opt/jre1.7.0_40/lib/i386/server/libjvm.so@
 
-Then type @python -V@ and check that it is version 2.5 or later (and not Python 3).
+Then type @python -V@ and check that it is version 2.5 or later (and not Python 3). Ideally this should be v2.7.6 or later as there have been some reported issues with earlier versions.
 
 If you would rather not put these versions in the PATH (e.g. you don't want to override the default version of Python in your Linux distribution), you can tell Stackdump which Java and/or Python to use explicitly by creating a file named @JAVA_CMD@ or @PYTHON_CMD@ respectively in the Stackdump root directory, and placing the path to the executable in there.
 
@@ -87,15 +129,15 @@ To start the import process, execute the following command -
 
 @stackdump_dir/manage.sh import_site --base-url site_url --dump-date dump_date path_to_xml_files@
 
-... where site_url is the URL of the site you're importing, e.g. __android.stackexchange.com__; dump_date is the date of the data dump you're importing, e.g. __August 2012__, and finally path_to_xml_files is the path to the XML files you just extracted. The dump_date is a text string that is shown in the app only, so it can be in any format you want.
+... where @site_url@ is the URL of the site you're importing, e.g. __android.stackexchange.com__; @dump_date@ is the date of the data dump you're importing, e.g. __August 2012__, and finally @path_to_xml_files@ is the path to the directory containing the XML files that were just extracted. The @dump_date@ is a text string that is shown in the app only, so it can be in any format you want.
 
-For example, to import the August 2012 data dump of the Android StackExchange site, you would execute -
+For example, to import the August 2012 data dump of the Android StackExchange site, with the files extracted into @/tmp/android@, you would execute -
 
 @stackdump_dir/manage.sh import_site --base-url android.stackexchange.com --dump-date "August 2012" /tmp/android@
 
 It is normal to get messages about unknown PostTypeIds and missing comments and answers. These errors are likely due to those posts being hidden via moderation.
 
-This can take anywhere between a minute to 10 hours or more depending on the site you're importing. As a rough guide, __android.stackexchange.com__ took a minute on my VM, while __stackoverflow.com__ took just over 10 hours.
+This can take anywhere between a minute to 20 hours or more depending on the site you're importing. As a rough guide, __android.stackexchange.com__ took a minute on my VM, while __stackoverflow.com__ took just under 24 hours.
 
 Repeat these steps for each site you wish to import. Do not attempt to import multiple sites at the same time; it will not work and you may end up with half-imported sites.
 
@@ -109,19 +151,53 @@ To start Stackdump, execute the following command -
 
 ... and visit port 8080 on that machine. That's it - your own offline, read-only instance of StackExchange.
 
-If you need to change the port that it runs on, modify @stackdump_dir/python/src/stackdump/settings.py@ and restart the app.
+If you need to change the port that it runs on, or modify other settings that control how Stackdump works; see the 'Optional configuration' section below for more details.
 
-The aforementioned @settings.py@ file also contains some other settings that control how Stackdump works.
+Both the search indexer and the app need to be running for Stackdump to work.
+
+h2. Optional configuration
+
+There are a few settings for those who like to tweak. There's no need to adjust them normally though; the default settings should be fine.
+
+The settings file is located in @stackdump_dir/python/src/stackdump/settings.py@. The web component will need to be restarted after changes have been made for them to take effect.
+
+* *SERVER_HOST* - the network interface to run the Stackdump web app on. Use _'0.0.0.0'_ for all interfaces, or _'127.0.0.1'_ for localhost only. By default, it runs on all interfaces.
+* *SERVER_PORT* - the port to run the Stackdump web app on. The default port is _8080_.
+* *SOLR_URL* - the URL to the Solr instance. The default assumes Solr is running on the same system. Change this if Solr is running on a different system.
+* *NUM_OF_DEFAULT_COMMENTS* - the number of comments shown by default for questions and answers before the remaining comments are hidden (and shown when clicked). The default is _3_ comments.
+* *NUM_OF_RANDOM_QUESTIONS* - the number of random questions shown on the home page of Stackdump and the site pages. The default is _3_ questions.
+* *REWRITE_LINKS_AND_IMAGES* - by default, all links are rewritten to either point internally or be marked as an external link, and image URLs are rewritten to point to a placeholder image. Set this setting to _False_ to disable this behaviour. 
+
+h2. Running Stackdump as a service
 
 Stackdump comes bundled with some init.d scripts as well which were tested on CentOS 5. These are located in the @init.d@ directory. To use these, you will need to modify them to specify the path to the Stackdump root directory and the user to run under.
 
-Both the search indexer and the app need to be running for Stackdump to work.
+Another option is to use "Supervisor":http://supervisord.org/ with a simple configuration file, e.g.,
+
+bc.. [program:stackdump-solr]
+command=/path/to/stackdump/start_solr.sh
+priority=900
+user=stackdump_user
+stopasgroup=true
+stdout_logfile=/path/to/stackdump/solr_stdout.log
+stderr_logfile=/path/to/stackdump/solr_stderr.log
+
+[program:stackdump-web]
+command=/path/to/stackdump/start_web.sh
+user=stackdump_user
+stopasgroup=true
+stdout_logfile=/path/to/stackdump/web_stdout.log
+stderr_logfile=/path/to/stackdump/web_stderr.log
+
+p. Supervisor v3.0b1 or later is required, due to the _stopasgroup_ parameter. Without this parameter, Supervisor will not be able to stop the Stackdump components properly as they're being executed from a script.
+
+Yet another option for those using newer Linux distributions is to create native "systemd service definitions":http://www.freedesktop.org/software/systemd/man/systemd.service.html of type _simple_ for each of the components.
 
 h2. Maintenance
 
 Stackdump stores all its data in the @data@ directory under its root directory. If you want to start fresh, just stop the app and the search indexer, delete that directory and restart the app and search indexer.
 
-To delete certain sites from Stackdump, use the manage_sites management command -
+To delete certain sites from Stackdump, use the @manage_sites@ management command -
 
 @stackdump_dir/manage.sh manage_sites -l@ to list the sites (and their site keys) currently in the system;
 @stackdump_dir/manage.sh manage_sites -d site_key@ to delete a particular site.
@@ -144,6 +220,7 @@ Stackdump leverages several open-source projects to do various things, including
 * "markdown":http://pypi.python.org/pypi/Markdown for rendering comments
 * "mathjax":http://www.mathjax.org/ for displaying mathematical expressions properly
 * "httplib2":http://code.google.com/p/httplib2/ as a dependency of pysolr
+* "requests":https://github.com/kennethreitz/requests/ as a dependency of pysolr
 * "Apache Solr":http://lucene.apache.org/solr/ for search functionality
 
 h2. Things not supported... yet

java/solr/server/solr/stackdump/conf/schema.xml

@@ -45,7 +45,7 @@
     that avoids logging every request
 -->
 
-<schema name="example" version="1.5">
+<schema name="stackdump" version="1.5">
   <!-- attribute "name" is the name of this schema and is only used for display purposes.
        version="x.y" is Solr's version number for the schema syntax and 
        semantics.  It should not normally be changed by applications.
@@ -110,6 +110,10 @@
 
    <!-- we'll get the values out of the JSON, so most fields are not stored -->
    <!-- fields are listed here so searches can be performed against them -->
+   <!-- this is used by Lucene to uniquely identify a post across all sites.
+        It is of the form "siteKey-id" and is necessary because post IDs are
+        reused across sites. -->
+   <field name="documentId" type="string" indexed="true" stored="true" required="true" />
    <!-- the ID field needs to be a string for the QueryElevationComponent -->
    <field name="id" type="string" indexed="true" stored="true" required="true" />
    <field name="siteKey" type="string" indexed="true" stored="true" required="true" />
@@ -196,7 +200,7 @@
  <!-- Field to use to determine and enforce document uniqueness. 
       Unless this field is marked with required="false", it will be a required field
    -->
- <uniqueKey>id</uniqueKey>
+ <uniqueKey>documentId</uniqueKey>
 
  <!-- DEPRECATED: The defaultSearchField is consulted by various query parsers when
   parsing a query string that isn't explicit about the field.  Machine (non-user)

manage.sh

@@ -35,5 +35,5 @@ else
     # shift off the command name so we don't pass it on
     shift
 
-    $SCRIPT_DIR/start_python.sh $command "$@"
+    $SCRIPT_DIR/start_python3.sh $command "$@"
 fi

python/packages/pysolr.py

@@ -246,6 +246,11 @@ class Solr(object):
     Optionally accepts ``timeout`` for wait seconds until giving up on a
     request. Default is ``60`` seconds.
 
+    Optionally accepts ``assume_clean`` to skip cleaning request of invalid XML
+    characters. This offers a slight performance improvement, but only set this
+    to ``True`` if you know your request is clean (e.g. coming from other XML
+    data). Bad things will happen otherwise. Default is ``False``.
+
     Usage::
 
         solr = pysolr.Solr('http://localhost:8983/solr')
@@ -253,10 +258,11 @@ class Solr(object):
         solr = pysolr.Solr('http://localhost:8983/solr', timeout=10)
 
     """
-    def __init__(self, url, decoder=None, timeout=60):
+    def __init__(self, url, decoder=None, timeout=60, assume_clean=False):
         self.decoder = decoder or json.JSONDecoder()
         self.url = url
         self.timeout = timeout
+        self.assume_clean = assume_clean
         self.log = self._get_log()
         self.session = requests.Session()
         self.session.stream = False
@@ -506,7 +512,10 @@ class Solr(object):
 
             value = "{0}".format(value)
 
-        return clean_xml_string(value)
+        if self.assume_clean:
+            return value
+        else:
+            return clean_xml_string(value)
 
     def _to_python(self, value):
         """

python/src/stackdump/app.py

@@ -26,7 +26,7 @@ import html5lib
 from html5lib.filters._base import Filter as HTML5LibFilterBase
 import markdown
 
-from stackdump.models import Site, Badge, Comment, User
+from stackdump.models import Site, Badge, User
 from stackdump import settings
 
 # STATIC VARIABLES
@@ -410,7 +410,8 @@ def view_question(site_key, question_id, answer_id=None):
     
     result = results.docs[0]
     convert_comments_to_html(result)
-    rewrite_result(result)
+    if settings.REWRITE_LINKS_AND_IMAGES:
+        rewrite_result(result)
     sort_answers(result)
     context['result'] = result
 
@@ -826,27 +827,31 @@ def _rewrite_html(html, app_url_root, sites_by_urls):
         internal_link = False
         url = t.get('href', None)
         if url:
-            host = urllib2.Request(url).get_host()
-            site = sites_by_urls.get(host, None)
-            if site:
-                # rewrite this URL for stackdump
-                question_id = SE_QUESTION_ID_RE.search(url)
-                if question_id:
-                    question_id = question_id.groupdict()['id']
-                    url = '%s%s/%s' % (app_url_root, site.key, question_id)
-                    t.set('href', url)
-                    t.set('class', t.get('class', '') + ' internal-link')
-                    internal_link = True
-                
-                answer_id = SE_ANSWER_ID_RE.search(url)
-                if answer_id:
-                    answer_id = answer_id.groupdict()['id']
-                    url = '%s%s/a/%s' % (app_url_root, site.key, answer_id)
-                    t.set('href', url)
-                    t.set('class', t.get('class', '') + ' internal-link')
-                    internal_link = True
+            try:
+                host = urllib2.Request(url).get_host()
+            except ValueError:
+                # invalid URL or local anchor, leaving as-is
+                pass
+            else:
+                site = sites_by_urls.get(host, None)
+                if site:
+                    # rewrite this URL for stackdump
+                    question_id = SE_QUESTION_ID_RE.search(url)
+                    if question_id:
+                        question_id = question_id.groupdict()['id']
+                        url = '%s%s/%s' % (app_url_root, site.key, question_id)
+                        t.set('href', url)
+                        internal_link = True
+                    answer_id = SE_ANSWER_ID_RE.search(url)
+                    if answer_id:
+                        answer_id = answer_id.groupdict()['id']
+                        url = '%s%s/a/%s' % (app_url_root, site.key, answer_id)
+                        t.set('href', url)
+                        internal_link = True
             
-            if not internal_link:
+            if internal_link:
+                t.set('class', t.get('class', '') + ' internal-link')
+            else:
                 t.set('class', t.get('class', '') + ' external-link')
     
     # get a string back

python/src/stackdump/commands/default_settings.py

@@ -0,0 +1 @@
+../default_settings.py

python/src/stackdump/commands/download_site_info.py

@@ -1,47 +1,142 @@
 #!/usr/bin/env python
 
 # This script downloads the sites RSS file and associated logos from the net.
-
-import urllib
+import tarfile
+import urllib.request
 from xml.etree import ElementTree
-import os
 import sys
+def printf(format, *args):
+    sys.stdout.write(format % args)
+from shutil import copy
+import os, ssl, fnmatch
+from optparse import OptionParser
+from xml.etree import ElementTree
+import elasticsearch
+
+import settings
+from sqlobject import sqlhub, connectionForURI,AND, IN, SQLObject, \
+    UnicodeCol, DateTimeCol, IntCol, DatabaseIndex, dbconnection
+from sqlobject.sqlbuilder import Delete, Insert
+from sqlobject.styles import DefaultStyle
+from pysolr import Solr, SolrError
+
+if (not os.environ.get('PYTHONHTTPSVERIFY', '') and
+        getattr(ssl, '_create_unverified_context', None)):
+    ssl._create_defat_https_context = ssl._create_unverified_context
+
+se_dir = os.path.join(os.environ.get('HOME'), 'stackexchange')
+sites_path = os.path.join(se_dir, 'Sites.xml')
 
 script_dir = os.path.dirname(sys.argv[0])
-sites_file_path = os.path.join(script_dir, '../../../../data/sites')
-
+sites_file_path = os.path.join(script_dir, ''
+                                           '../../../../data/')
 # ensure the data directory exists
+# download the sites RSS file
+
 if not os.path.exists(os.path.dirname(sites_file_path)):
     os.mkdir(os.path.dirname(sites_file_path))
 
-# download the sites RSS file
-print 'Downloading StackExchange sites RSS file...',
-urllib.urlretrieve('http://stackexchange.com/feeds/sites', sites_file_path)
-print 'done.'
+print('Downloading StackExchange sites XML file...')
+# urllib.request.urlretrieve('https://archive.org/download/stackexchange/Sites.xml', sites_file_path)
+print('done.')
 
-print ''
+print('')
 
-# parse sites RSS file and download logos
-logos_dir_path = os.path.join(script_dir, '../../../media/images/logos')
+
+
+# parse sites RSS file and download logosc
+images_dir_path = os.path.join(script_dir, '../../../media/images')
+print(os.listdir(images_dir_path))
+logos_dir_path = os.path.join(images_dir_path, 'logos48')
 if not os.path.exists(logos_dir_path):
     os.mkdir(logos_dir_path)
+icons_dir_path = os.path.join(images_dir_path, 'icons')
+if not os.path.exists(icons_dir_path):
+    os.mkdir(icons_dir_path)
+badges_dir_path = os.path.join(images_dir_path, 'badges')
+if not os.path.exists(badges_dir_path):
+    os.mkdir(badges_dir_path)
 
-with open(sites_file_path) as f:
+with open(sites_path) as f:
     sites_file = ElementTree.parse(f)
-    entries = sites_file.findall('{http://www.w3.org/2005/Atom}entry')
-    
-    for entry in entries:
-        entry_title = entry.find('{http://www.w3.org/2005/Atom}title').text.encode('ascii', 'ignore')
-        
+    sites = sites_file.findall('row')
+    # print(rows[0].attrib)
+
+    for site in sites:
+        site_title = site.attrib['LongName']
+        site_name = site.attrib['Name']
         # extract the key from the url - remove the http:// and .com
-        site_key = entry.find('{http://www.w3.org/2005/Atom}id').text
-        if site_key.startswith('http://'):
-            site_key = site_key[len('http://'):]
-        if site_key.endswith('.com'):
-            site_key = site_key[:-len('.com')]
-        if site_key.endswith('.stackexchange'):
-            site_key = site_key[:-len('.stackexchange')]
-        
-        print 'Downloading logo for %s...' % entry_title,
-        urllib.urlretrieve('http://sstatic.net/%s/img/icon-48.png' % site_key, os.path.join(logos_dir_path, '%s.png' % site_key))
-        print 'done.'
+        site_key = site.attrib['TinyName']
+        site_url = site.attrib['Url'][8:]
+        logo_url = site.attrib['ImageUrl']
+        icon_url = site.attrib['IconUrl']
+        badge_url = site.attrib['BadgeIconUrl']
+
+
+        site_vars = (site_url, site_key, site_name, site_title)
+        # print(site_vars)
+        printf('Site: %s,  key=%s, name="%s", longname="%s"\n' % site_vars)
+        try:
+            logo_file = os.path.join(logos_dir_path, 'logo-%s.png' % site_key)
+            if not os.path.exists(logo_file):
+                print('Downloading logo for %s...' % site_title, urllib.request.urlretrieve(logo_url, logo_file))
+        except Exception as e:
+            print('Failed download logo for %s...' % site_title, str(e))
+
+        try:
+            icon_path = os.path.join(icons_dir_path, 'icon-%s.png' % site_key)
+            if not os.path.exists(icon_path):
+                print('Downloading icon for %s...' % site_title, urllib.request.urlretrieve(icon_url, icon_path))
+        except:
+            print('Failed download ico for %s...' % site_title, icon_url)
+
+        try:
+            badge_file = os.path.join(badges_dir_path, 'badge-%s.png' % site_key)
+            if not os.path.exists(icon_path):
+                print('Downloading badge for %s...' % site_title, urllib.request.urlretrieve(badge_url, badge_file))
+        except:
+            printf('Failed download badge for %s...' % site_title)
+
+        site_files = []
+        print('Key: ' + site_url)
+        for root, dirs, files in os.walk(se_dir):
+            for name in files:
+                if fnmatch.fnmatch(name, site_url + '*'):
+                    print('Match: ' + os.path.join(root, name))
+                    site_files.append(os.path.join(root, name))
+
+        sites_data = sites_file_path
+        for site_file in site_files:
+            dst = sites_data + os.sep + site_key[0] + os.sep + site_key + os.sep + 'xml'
+            os.makedirs(dst, exist_ok=True)
+            os.chdir(dst)
+            os.system('tar xzf '+site_file)
+            print('Data: ' + site_file)
+
+def prepare_site(xml_root, dump_date, site_key):
+    print('Using the XML root path: ' + xml_root + '\n')
+
+    if not os.path.exists(xml_root):
+        print('The given XML root path does not exist.')
+        sys.exit(1)
+
+    # connect to the database
+    print('Connecting to the Stackdump database...')
+    conn_str = settings.DATABASE_CONN_STR
+    sqlhub.processConnection = connectionForURI(conn_str)
+    print('Connected.\n')
+
+# MAIN METHOD
+if __name__ == '__main__':
+    parser = OptionParser(usage='usage: %pro'
+                                'g [options] xml_root_dir')
+    parser.add_option('-k', '--site-key', help='Key of the site (if not in sites).')
+    parser.add_option('-c', '--dump-date', help='Dump date of the site.')
+
+    (cmd_options, cmd_args) = parser.parse_args()
+
+    if len(cmd_args) < 1:
+        print('The path to the directory containing the extracted XML files is required.')
+        sys.exit(1)
+
+    prepare_site(cmd_args[0], cmd_options.dump_date, cmd_options.site_key)

python/src/stackdump/commands/settings.py

@@ -0,0 +1 @@
+../settings.py

python/src/stackdump/default_settings.py

@@ -18,7 +18,8 @@ SERVER_PORT = 8080
 SOLR_URL = 'http://localhost:8983/solr/stackdump/'
 
 import os
-DATABASE_CONN_STR = 'sqlite://%s/../../../data/stackdump.sqlite' % os.path.dirname(__file__)
+DATABASE_CONN_STR = 'sqlite:///' + os.path.join(os.path.dirname(__file__), '..', '..', '..', 'data', 'stackdump.sqlite')
+TEMP_COMMENTS_DATABASE_DIR = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'data')
 
 # if the website is hosted under a subpath, specify it here. It must end with a
 # slash.
@@ -31,6 +32,9 @@ NUM_OF_DEFAULT_COMMENTS = 3
 # number of random questions to show on search query pages
 NUM_OF_RANDOM_QUESTIONS = 3
 
+# rewrite links and images to point internally or to a placeholder respectively
+REWRITE_LINKS_AND_IMAGES = True
+
 # settings that are available in templates
 TEMPLATE_SETTINGS = [
     'APP_URL_ROOT',

python/src/stackdump/models.py

@@ -5,6 +5,10 @@
 from sqlobject import SQLObject, UnicodeCol, DateTimeCol, IntCol, ForeignKey, \
                       DatabaseIndex
 
+
+ISO_DATE_FORMAT = '%Y-%m-%dT%H:%M:%S.%f'
+
+
 class Site(SQLObject):
     name = UnicodeCol()
     desc = UnicodeCol()
@@ -15,34 +19,23 @@ class Site(SQLObject):
     
     siteKey_index = DatabaseIndex(key, unique=True)
 
+
 class Badge(SQLObject):
     sourceId = IntCol()
     site = ForeignKey('Site', cascade=True)
     userId = IntCol()
     name = UnicodeCol()
-    date = DateTimeCol()
+    date = DateTimeCol(datetimeFormat=ISO_DATE_FORMAT)
 
-class Comment(SQLObject):
-    sourceId = IntCol()
-    site = ForeignKey('Site', cascade=True)
-    postId = IntCol()
-    score = IntCol()
-    text = UnicodeCol()
-    creationDate = DateTimeCol()
-    userId = IntCol()
-    
-    siteId_postId_index = DatabaseIndex(site, postId)
-    
-    json_fields = [ 'id', 'score', 'text', 'creationDate', 'userId' ]
 
 class User(SQLObject):
     sourceId = IntCol()
     site = ForeignKey('Site', cascade=True)
     reputation = IntCol()
-    creationDate = DateTimeCol()
+    creationDate = DateTimeCol(datetimeFormat=ISO_DATE_FORMAT)
     displayName = UnicodeCol()
     emailHash = UnicodeCol()
-    lastAccessDate = DateTimeCol()
+    lastAccessDate = DateTimeCol(datetimeFormat=ISO_DATE_FORMAT)
     websiteUrl = UnicodeCol()
     location = UnicodeCol()
     age = IntCol()

python/src/stackdump/settings.py

@@ -19,9 +19,10 @@ from default_settings import *
 # uncomment if the default host and port for Solr is different.
 #SOLR_URL = 'http://localhost:8983/solr/stackdump/'
 
-# uncomment if the database for Stackdump is not the default SQLite one
-#import os
-#DATABASE_CONN_STR = 'sqlite://%s/../../../data/stackdump.sqlite' % os.path.dirname(__file__)
+# uncomment if the database for Stackdump is not the default SQLite one or you
+# wish to have the database at a different path to the stackdump_root/data
+# directory
+#DATABASE_CONN_STR = 'sqlite:///' + path_to_the_database
 
 # if the website is hosted under a subpath, specify it here. It must end with a
 # slash.
@@ -33,3 +34,6 @@ from default_settings import *
 
 # number of random questions to show on search query pages
 #NUM_OF_RANDOM_QUESTIONS = 3
+
+# rewrite links and images to point internally or to a placeholder respectively
+#REWRITE_LINKS_AND_IMAGES = True

start_python3.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+##
+# This script attempts to find a version of Python on the system PATH, and
+# checks that it is 2.5+.
+#
+# A alternate Python command can be specified in a file named PYTHON_CMD in this
+# script's directory. This path will override any lookup on the system PATH.
+##
+
+# FUNCTIONS
+function checkPythonVersion {
+    if [ ! -z "$1" ]
+    then
+        PYTHON_VER_MAJOR=`echo $1 | cut -d "." -f 1`
+        PYTHON_VER_MINOR=`echo $1 | cut -d "." -f 2`
+        
+        if [ $PYTHON_VER_MAJOR -eq "3" -a $PYTHON_VER_MINOR -ge "5" ]
+        then
+            return 1
+        fi
+    fi
+    
+    return 0
+}
+
+# MAIN
+SCRIPT_DIR=`dirname $0`
+PYTHON_CMD=python3
+
+# if there is a PYTHON_CMD file in the script directory, use that instead
+if [ -e "$SCRIPT_DIR/PYTHON_CMD" ]
+then
+    PYTHON_CMD=`cat "$SCRIPT_DIR/PYTHON_CMD"`
+fi
+
+if [ ! -z "`which "$PYTHON_CMD" 2>/dev/null`" ]
+then
+    # check if Python is the right version
+    PYTHON_VER=`"$PYTHON_CMD" -V 2>&1 | cut -d " " -f 2`
+    checkPythonVersion "$PYTHON_VER"
+    if [ $? == 1 ]
+    then
+        echo "Using Python `which "$PYTHON_CMD"`"
+        
+        # execution ends here if Python is found
+        PYTHONPATH=$SCRIPT_DIR/pyth3/packages:$SCRIPT_DIR/python/src:$PYTHONPATH
+        env "PYTHONPATH=$PYTHONPATH" "$PYTHON_CMD" "$@"
+        exit $?
+    fi
+fi
+
+# if we get here, it means the right version of Python was not found
+echo 'No suitable version of Python was found. Python 2.5 or later is required.'

start_solr.sh

@@ -22,4 +22,4 @@ then
 fi
 
 cd "$SCRIPT_DIR/java/solr/server"
-"$JAVA_CMD" -Xmx2048M -XX:MaxPermSize=512M -jar start.jar
+"$JAVA_CMD" -Xmx2048M -XX:MaxPermSize=512M -Djetty.host=127.0.0.1 -jar start.jar