dekker.one
A web scraper build to search specific information for a given compound (and its pseudonyms)
changes in sourceloader such that it is possible to create an executable in Windows, also the setup.py file to create the executable
+17
-6
+4
-1
setup.py
+4
-1
setup.py
···
1
1
import sys
2
2
from cx_Freeze import setup, Executable
3
3
4
+
# After running the setup file (python setup.py build) the scrapy/VERSION file has to be manually put into the
5
+
# library.zip, also the FourmiCrawler map has to be copied to both the library and the exe.win32-2.7 folder. after
6
+
# putting the files in the library the library has to be zipped and replace the old library.
4
7
# Dependencies are automatically detected, but it might need fine tuning.
5
-
build_exe_options = {"packages": ["os", "scrapy", "lxml._elementpath"], "excludes": ["tkinter"]}
8
+
build_exe_options = {"packages": ["os", "scrapy", "lxml", "w3lib", "pkg_resources", "zope.interface"], "excludes": []}
6
9
7
10
# GUI applications require a different base on Windows (the default is for a
8
11
# console application).
+13
-5
sourceloader.py
+13
-5
sourceloader.py
···
1
1
import inspect
2
+
import sys
2
3
import os
3
4
import re
4
5
from FourmiCrawler.sources.source import Source
···
8
9
sources = []
9
10
10
11
def __init__(self, rel_dir="FourmiCrawler\\sources"):
11
-
path = os.path.dirname(os.path.abspath(__file__))
12
-
path += "\\" + rel_dir
13
-
known_parser = set()
12
+
13
+
if hasattr(sys,'frozen'):
14
+
path = os.path.dirname(sys.executable)
15
+
path += "\\" + rel_dir
16
+
known_parser = set()
17
+
18
+
else:
19
+
path = os.path.dirname(os.path.abspath(__file__))
20
+
path += "\\" + rel_dir
21
+
known_parser = set()
14
22
15
23
for py in [f[:-3] for f in os.listdir(path) if f.endswith('.py') and f != '__init__.py']:
16
24
mod = __import__('.'.join([rel_dir.replace('\\', "."), py]), fromlist=[py])
17
25
classes = [getattr(mod, x) for x in dir(mod) if inspect.isclass(getattr(mod, x))]
18
26
for cls in classes:
19
27
if issubclass(cls, Source) and cls not in known_parser:
20
-
self.sources.append(cls()) # [review] - Would we ever need arguments for the parsers?
21
-
known_parser.add(cls)
28
+
self.sources.append(cls()) # [review] - Would we ever need arguments for the parsers?
29
+
# known_parser.add(cls)
22
30
23
31
def include(self, source_names):
24
32
new = set()