+15

.travis.yml

··· 1 1 + # Config file for automatic testing at travis-ci.org 2 2 + 3 3 + language: python 4 4 + python: 2.7 5 5 + 6 6 + # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors 7 7 + install: 8 8 + - pip install Scrapy docopt 9 9 + 10 10 + # command to run tests, e.g. python setup.py test 11 11 + script: 12 12 + - nosetests tests 13 13 + 14 14 + notifications: 15 15 + slack: descartes2:6sgCzx3PvrO9IIMwKxj12dDM

+10 -8

FourmiCrawler/pipelines.py

··· 4 4 5 5 from scrapy.exceptions import DropItem 6 6 7 7 - class RemoveNonePipeline(object): 8 7 8 8 + class RemoveNonePipeline(object): 9 9 def __init__(self): 10 10 - self.known_values = set() 10 10 + pass 11 11 12 12 - def process_item(self, item, spider): 12 12 + @staticmethod 13 13 + def process_item(item, spider): 13 14 """ 14 15 Processing the items so None values are replaced by empty strings 15 16 :param item: The incoming item ··· 21 22 item[key] = "" 22 23 return item 23 24 24 24 - class DuplicatePipeline(object): 25 25 26 26 + class DuplicatePipeline(object): 26 27 def __init__(self): 27 28 self.known_values = set() 28 29 ··· 35 36 """ 36 37 value = (item['attribute'], item['value'], item['conditions']) 37 38 if value in self.known_values: 38 38 - raise DropItem("Duplicate item found: %s" % item) # #[todo] append sources of first item. 39 39 + raise DropItem("Duplicate item found: %s" % item) # [todo] append sources of first item. 39 40 else: 40 41 self.known_values.add(value) 41 42 return item 43 43 + 42 44 43 45 class AttributeSelectionPipeline(object): 44 44 - 45 46 def __init__(self): 46 46 - pass; 47 47 + pass 47 48 48 48 - def process_item(self, item, spider): 49 49 + @staticmethod 50 50 + def process_item(item, spider): 49 51 """ 50 52 The items are processed using the selected attribute list available in the spider, 51 53 items that don't match the selected items are dropped.

+1 -1

FourmiCrawler/settings.py

··· 3 3 # For simplicity, this file contains only the most important settings by 4 4 # default. All the other settings are documented here: 5 5 # 6 6 - # http://doc.scrapy.org/en/latest/topics/settings.html 6 6 + # http://doc.scrapy.org/en/latest/topics/settings.html 7 7 # 8 8 9 9 BOT_NAME = 'FourmiCrawler'

+6 -5

FourmiCrawler/sources/ChemSpider.py

··· 1 1 - from source import Source 1 1 + import re 2 2 + 2 3 from scrapy import log 3 4 from scrapy.http import Request 4 5 from scrapy.selector import Selector 6 6 + 7 7 + from source import Source 5 8 from FourmiCrawler.items import Result 6 6 - import re 9 9 + 7 10 8 11 # [TODO] - Maybe clean up usage of '.extract()[0]', because of possible IndexError exception. 9 12 ··· 58 61 prop_conditions = '' 59 62 60 63 # Test for properties without values, with one hardcoded exception 61 61 - if (not re.match(r'^\d', prop_value) or 62 62 - (prop_name == 'Polarizability' and 63 63 - prop_value == '10-24cm3')): 64 64 + if not re.match(r'^\d', prop_value) or (prop_name == 'Polarizability' and prop_value == '10-24cm3'): 64 65 continue 65 66 66 67 # Match for condition in parentheses

+13 -10

FourmiCrawler/sources/NIST.py

··· 1 1 - from source import Source 1 1 + import re 2 2 + 2 3 from scrapy import log 3 4 from scrapy.http import Request 4 5 from scrapy.selector import Selector 6 6 + 7 7 + from source import Source 5 8 from FourmiCrawler.items import Result 6 6 - import re 9 9 + 7 10 8 11 # [TODO]: values can be '128.', perhaps remove the dot in that case? 9 12 # [TODO]: properties have references and comments which do not exist in the 10 10 - # Result item, but should be included eventually. 13 13 + # Result item, but should be included eventually. 11 14 12 15 class NIST(Source): 13 16 """NIST Scraper plugin ··· 15 18 This plugin manages searching for a chemical on the NIST website 16 19 and parsing the resulting page if the chemical exists on NIST. 17 20 """ 18 18 - website = "http://webbook.nist.gov/*" 21 21 + website = "http://webbook.nist.gov/*" 19 22 20 23 search = 'cgi/cbook.cgi?Name=%s&Units=SI&cTP=on' 21 24 ··· 75 78 requests.extend(self.parse_generic_data(table, summary)) 76 79 else: 77 80 log.msg('NIST table: NOT SUPPORTED', level=log.WARNING) 78 78 - continue #Assume unsupported 81 81 + continue # Assume unsupported 79 82 return requests 80 83 81 84 def parse_generic_info(self, sel): ··· 103 106 data['IUPAC Standard InChI'] = raw_inchi.extract()[0] 104 107 105 108 raw_inchikey = ul.xpath('li[strong="IUPAC Standard InChIKey:"]' 106 106 - '/tt/text()') 109 109 + '/tt/text()') 107 110 data['IUPAC Standard InChIKey'] = raw_inchikey.extract()[0] 108 111 109 112 raw_cas_number = ul.xpath('li[strong="CAS Registry Number:"]/text()') ··· 129 132 results = [] 130 133 for tr in table.xpath('tr[td]'): 131 134 extra_data_url = tr.xpath('td[last()][a="Individual data points"]' 132 132 - '/a/@href').extract() 135 135 + '/a/@href').extract() 133 136 if extra_data_url: 134 137 request = Request(url=self.website[:-1] + extra_data_url[0], 135 135 - callback=self.parse_individual_datapoints) 138 138 + callback=self.parse_individual_datapoints) 136 139 results.append(request) 137 140 continue 138 141 data = [] ··· 179 182 'conditions': '%s K, (%s -> %s)' % (tds[1], tds[2], tds[3]) 180 183 }) 181 184 results.append(result) 182 182 - 183 185 184 186 return results 185 187 ··· 228 230 229 231 return results 230 232 231 231 - def parse_individual_datapoints(self, response): 233 233 + @staticmethod 234 234 + def parse_individual_datapoints(response): 232 235 """Parses the page linked from aggregate data""" 233 236 sel = Selector(response) 234 237 table = sel.xpath('//table[@class="data"]')[0]

+5 -3

FourmiCrawler/sources/WikipediaParser.py

··· 1 1 + import re 2 2 + 1 3 from scrapy.http import Request 2 4 from scrapy import log 3 3 - from source import Source 4 5 from scrapy.selector import Selector 6 6 + 7 7 + from source import Source 5 8 from FourmiCrawler.items import Result 6 6 - import re 7 9 8 10 9 11 class WikipediaParser(Source): ··· 36 38 """ scrape data from infobox on wikipedia. """ 37 39 items = [] 38 40 39 39 - #be sure to get chembox (wikipedia template) 41 41 + # be sure to get chembox (wikipedia template) 40 42 tr_list = sel.xpath('.//table[@class="infobox bordered"]//td[not(@colspan)]'). \ 41 43 xpath('normalize-space(string())') 42 44 prop_names = tr_list[::2]

+12 -10

FourmiCrawler/spider.py

··· 9 9 A spider writen for the Fourmi Project which calls upon all available sources to request and scrape data. 10 10 """ 11 11 name = "FourmiSpider" 12 12 - __sources = [] 13 13 - synonyms = [] 12 12 + _sources = [] 13 13 + synonyms = set() 14 14 15 15 def __init__(self, compound=None, selected_attributes=[".*"], *args, **kwargs): 16 16 """ ··· 19 19 :param selected_attributes: A list of regular expressions that the attributes should match. 20 20 """ 21 21 super(FourmiSpider, self).__init__(*args, **kwargs) 22 22 - self.synonyms.append(compound) 23 23 - self.selected_attributes = selected_attributes; 22 22 + self.synonyms.add(compound) 23 23 + self.selected_attributes = selected_attributes 24 24 25 25 def parse(self, response): 26 26 """ ··· 29 29 :param response: A Scrapy Response object that should be parsed 30 30 :return: A list of Result items and new Request to be handled by the scrapy core. 31 31 """ 32 32 - for source in self.__sources: 32 32 + for source in self._sources: 33 33 if re.match(source.website, response.url): 34 34 log.msg("Url: " + response.url + " -> Source: " + source.website, level=log.DEBUG) 35 35 return source.parse(response) ··· 42 42 :return: A list of Scrapy Request objects 43 43 """ 44 44 requests = [] 45 45 - for parser in self.__sources: 46 46 - parser_requests = parser.new_compound_request(compound) 47 47 - if parser_requests is not None: 48 48 - requests.append(parser_requests) 45 45 + if compound not in self.synonyms: 46 46 + self.synonyms.add(compound) 47 47 + for parser in self._sources: 48 48 + parser_requests = parser.new_compound_request(compound) 49 49 + if parser_requests is not None: 50 50 + requests.append(parser_requests) 49 51 return requests 50 52 51 53 def start_requests(self): ··· 71 73 A function add a new Parser object to the list of available parsers. 72 74 :param source: A Source Object 73 75 """ 74 74 - self.__sources.append(source) 76 76 + self._sources.append(source) 75 77 source.set_spider(self)

+4

README.md

··· 1 1 # Fourmi 2 2 3 3 + **Master branch**: [](https://travis-ci.org/Recondor/Fourmi) 4 4 + 5 5 + **Developing branch**: [](https://travis-ci.org/Recondor/Fourmi) 6 6 + 3 7 Fourmi is an web scraper for chemical substances. The program is designed to be 4 8 used as a search engine to search multiple chemical databases for a specific 5 9 substance. The program will produce all available attributes of the substance

+1 -1

fourmi.py

··· 1 1 - #!/usr/bin/env python 1 1 + # !/usr/bin/env python 2 2 """ 3 3 Fourmi, a web scraper build to search specific information for a given compound (and it's pseudonyms). 4 4

+1

tests/__init__.py

··· 1 1 +

+52

tests/test_pipeline.py

··· 1 1 + import copy 2 2 + import unittest 3 3 + 4 4 + from scrapy.exceptions import DropItem 5 5 + 6 6 + from FourmiCrawler import pipelines, spider, items 7 7 + 8 8 + 9 9 + class TestPipelines(unittest.TestCase): 10 10 + def setUp(self): 11 11 + self.testItem = items.Result() 12 12 + 13 13 + def test_none_pipeline(self): 14 14 + # Testing the pipeline that replaces the None values in items. 15 15 + self.testItem["value"] = "abc" 16 16 + pipe = pipelines.RemoveNonePipeline() 17 17 + processed = pipe.process_item(self.testItem, spider.FourmiSpider()) 18 18 + 19 19 + self.assertTrue(processed["value"] == "abc") 20 20 + 21 21 + for key in self.testItem: 22 22 + self.assertIsNotNone(processed[key]) 23 23 + if key is not "value": 24 24 + self.assertIs(processed[key], "") 25 25 + 26 26 + def test_duplicate_pipeline(self): 27 27 + # Testing the pipeline that removes duplicates. 28 28 + self.testItem["attribute"] = "test" 29 29 + self.testItem["value"] = "test" 30 30 + self.testItem["conditions"] = "test" 31 31 + 32 32 + pipe = pipelines.DuplicatePipeline() 33 33 + self.assertEqual(pipe.process_item(self.testItem, spider.FourmiSpider()), self.testItem) 34 34 + self.assertRaises(DropItem, pipe.process_item, self.testItem, spider.FourmiSpider()) 35 35 + 36 36 + other_item = copy.deepcopy(self.testItem) 37 37 + other_item["value"] = "test1" 38 38 + self.assertEqual(pipe.process_item(other_item, spider.FourmiSpider()), other_item) 39 39 + 40 40 + def test_attribute_selection(self): 41 41 + # Testing the pipeline that selects attributes. 42 42 + item1 = copy.deepcopy(self.testItem) 43 43 + item2 = copy.deepcopy(self.testItem) 44 44 + 45 45 + item1["attribute"] = "abd" 46 46 + item2["attribute"] = "abc" 47 47 + 48 48 + s = spider.FourmiSpider(selected_attributes=["a.d"]) 49 49 + pipe = pipelines.AttributeSelectionPipeline() 50 50 + 51 51 + self.assertEqual(pipe.process_item(item1, s), item1) 52 52 + self.assertRaises(DropItem, pipe.process_item, item2, s)

+33

tests/test_sourceloader.py

··· 1 1 + import unittest 2 2 + 3 3 + from sourceloader import SourceLoader 4 4 + 5 5 + 6 6 + class TestSourceloader(unittest.TestCase): 7 7 + def setUp(self): 8 8 + self.loader = SourceLoader() 9 9 + 10 10 + def test_init(self): 11 11 + # Test if sourceloader points to the right directory, where the sources are present. 12 12 + self.assertIn("Source: Source", str(self.loader)) 13 13 + self.assertIn("Source: NIST", str(self.loader)) 14 14 + self.assertIn("Source: ChemSpider", str(self.loader)) 15 15 + self.assertIn("Source: WikipediaParser", str(self.loader)) 16 16 + 17 17 + def test_include(self): 18 18 + # Tests for the include functionality. 19 19 + self.loader.include(["So.rc.*"]) 20 20 + 21 21 + self.assertIn("Source: Source", str(self.loader)) 22 22 + self.assertNotIn("Source: NIST", str(self.loader)) 23 23 + self.assertNotIn("Source: ChemSpider", str(self.loader)) 24 24 + self.assertNotIn("Source: WikipediaParser", str(self.loader)) 25 25 + 26 26 + def test_exclude(self): 27 27 + # Tests for the exclude functionality. 28 28 + self.loader.exclude(["So.rc.*"]) 29 29 + 30 30 + self.assertNotIn("Source: Source", str(self.loader)) 31 31 + self.assertIn("Source: NIST", str(self.loader)) 32 32 + self.assertIn("Source: ChemSpider", str(self.loader)) 33 33 + self.assertIn("Source: WikipediaParser", str(self.loader))

+61

tests/test_spider.py

··· 1 1 + import unittest 2 2 + 3 3 + from scrapy.http import Request 4 4 + 5 5 + from FourmiCrawler import spider 6 6 + from FourmiCrawler.sources.ChemSpider import ChemSpider 7 7 + from FourmiCrawler.sources.source import Source 8 8 + 9 9 + 10 10 + class TestFoumiSpider(unittest.TestCase): 11 11 + def setUp(self): 12 12 + self.compound = "test_compound" 13 13 + self.attributes = ["a.*", ".*a"] 14 14 + self.spi = spider.FourmiSpider(self.compound, self.attributes) 15 15 + 16 16 + def test_init(self): 17 17 + # Test the initiation of the Fourmi spider 18 18 + self.assertIn(self.compound, self.spi.synonyms) 19 19 + for attr in self.attributes: 20 20 + self.assertIn(attr, self.spi.selected_attributes) 21 21 + 22 22 + def test_add_source(self): 23 23 + # Testing the source adding function of the Fourmi spider 24 24 + src = Source() 25 25 + self.spi.add_source(src) 26 26 + self.assertIn(src, self.spi._sources) 27 27 + 28 28 + def test_add_sources(self): 29 29 + # Testing the function that adds multiple sources 30 30 + srcs = [Source(), Source(), Source()] 31 31 + self.spi.add_sources(srcs) 32 32 + 33 33 + for src in srcs: 34 34 + self.assertIn(src, self.spi._sources) 35 35 + 36 36 + def test_start_requests(self): 37 37 + # A test for the function that generates the start requests 38 38 + self.spi._sources = [] 39 39 + 40 40 + src = Source() 41 41 + self.spi.add_source(src) 42 42 + self.assertEqual(self.spi.start_requests(), []) 43 43 + 44 44 + src2 = ChemSpider() 45 45 + self.spi.add_source(src2) 46 46 + self.assertIsNotNone(self.spi.start_requests()) 47 47 + 48 48 + def test_synonym_requests(self): 49 49 + # A test for the synonym request function 50 50 + self.spi._sources = [] 51 51 + 52 52 + src = Source() 53 53 + self.spi.add_source(src) 54 54 + self.assertEqual(self.spi.get_synonym_requests("new_compound"), []) 55 55 + self.assertIn("new_compound", self.spi.synonyms) 56 56 + 57 57 + src2 = ChemSpider() 58 58 + self.spi.add_source(src2) 59 59 + self.assertIsInstance(self.spi.get_synonym_requests("other_compound")[0], Request) 60 60 + self.assertIn("other_compound", self.spi.synonyms) 61 61 + self.assertEqual(self.spi.get_synonym_requests("other_compound"), [])